20 Newsgroups Data: Convolutional Neural Network (CNN) for Global Vector (GloVe) Word Embeddings

In [1]:
'''This script loads pre-trained word embeddings (GloVe embeddings)
into a frozen Keras Embedding layer, and uses it to
train a text classification model on the 20 Newsgroup dataset
(classication of newsgroup messages into 20 different categories).

GloVe embedding data can be found at:
http://nlp.stanford.edu/data/glove.6B.zip
(source page: http://nlp.stanford.edu/projects/glove/)

20 Newsgroup data can be found at:
http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html
'''

from __future__ import print_function

import os
import sys
import numpy as np
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.utils import to_categorical
from keras.layers import Dense, Input, Flatten
from keras.layers import Conv1D, MaxPooling1D, Embedding
from keras.models import Model

Using CNTK backend
In [2]:
BASE_DIR = 'keras/examples'
GLOVE_DIR = BASE_DIR + '/glove.6B/'
TEXT_DATA_DIR = BASE_DIR + '/20_newsgroup/'
MAX_SEQUENCE_LENGTH = 1000
MAX_NB_WORDS = 20000
EMBEDDING_DIM = 100
VALIDATION_SPLIT = 0.2
In [3]:
# first, build index mapping words in the embeddings set
# to their embedding vector

print('Indexing word vectors.')

embeddings_index = {}
f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

print('Found %s word vectors.' % len(embeddings_index))

Indexing word vectors.
Found 400000 word vectors.
In [4]:
# second, prepare text samples and their labels
print('Processing text dataset')

texts = []  # list of text samples
labels_index = {}  # dictionary mapping label name to numeric id
labels = []  # list of label ids
for name in sorted(os.listdir(TEXT_DATA_DIR)):
    path = os.path.join(TEXT_DATA_DIR, name)
    if os.path.isdir(path):
        label_id = len(labels_index)
        labels_index[name] = label_id
        for fname in sorted(os.listdir(path)):
            if fname.isdigit():
                fpath = os.path.join(path, fname)
                if sys.version_info < (3,):
                    f = open(fpath)
                else:
                    f = open(fpath, encoding='latin-1')
                t = f.read()
                i = t.find('\n\n')  # skip header
                if 0 < i:
                    t = t[i:]
                texts.append(t)
                f.close()
                labels.append(label_id)

print('Found %s texts.' % len(texts))

Processing text dataset
Found 19997 texts.
In [5]:
# finally, vectorize the text samples into a 2D integer tensor
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))

encoded_text = sequences[10000]    # 20_newsgroup/rec.sport.hockey/52550
index_to_word = dict([(value, key) for (key, value) in word_index.items()])
decoded_text = ' '.join([index_to_word.get(i, '?') for i in encoded_text])
print(encoded_text)
print(decoded_text)

data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)

labels = to_categorical(np.asarray(labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)

Found 174074 unique tokens.
[2941, 183, 585, 29, 102, 1479, 26, 43, 7, 355, 67, 1, 2667, 216, 3800, 2840, 39, 25, 397, 10624, 207, 207, 5, 944, 25, 1756, 3256, 5169, 139, 1, 150, 218, 12, 921, 240, 1064, 366, 38, 18, 335, 4, 5192, 3865, 6, 7481, 23, 1, 366, 1078, 139, 17, 3027, 6, 1, 195, 177, 3556, 454, 1543, 3511, 20, 1427, 776, 347, 85, 8760, 1, 2667, 3511, 20, 2276, 2126, 2213, 2396, 3314, 2213, 2477, 5, 3314, 776, 20, 4, 602, 3, 2621, 12050, 3321, 6, 11, 25, 31, 14, 180, 1, 2667, 7481, 25, 871, 4017, 801, 62, 3, 489, 350, 27, 4107, 206, 1, 2308, 164, 2961, 80, 42, 13, 851, 4, 7920, 337, 366, 3598, 7481, 2976, 67, 27, 690, 83, 2, 1, 2308, 6, 92, 198, 3800, 3556, 8, 1, 2288, 2119, 13, 105, 122, 4, 238, 9, 1, 12305, 3, 32, 8227, 366, 9, 38, 122, 1, 2308, 8, 1, 310, 2245, 13, 104, 1162, 11, 14, 8, 615, 1, 238, 7, 25, 178, 2, 122, 24, 7, 84, 1, 310, 8, 3556, 81, 4, 10274, 180, 752, 5, 25, 1, 126, 314, 5169, 2, 503, 1, 6923, 5, 5449, 164, 3784, 66, 1367, 62, 2, 1222, 166, 464, 7, 41, 2129, 9, 4, 1786, 64, 3800, 41, 1517, 1269, 114, 23, 1, 366, 7465, 24, 14, 8, 723, 888, 76, 7, 169, 42, 17, 84, 1, 7481, 1419, 41, 192, 555, 1, 366, 8, 4, 2151, 7356, 8227, 5944, 8489, 40, 1543, 25, 169, 390, 184, 48, 66, 2745, 2, 283, 77, 23, 27, 3389, 3421, 2, 130, 361, 776, 1390, 18, 8462, 3933, 12, 58, 3, 1, 66, 8246, 174, 1, 86, 1, 347, 195, 2477, 3036, 1, 2667, 81, 3036, 776, 17, 3314, 572, 33, 746, 454, 29, 1, 1516, 1812, 5, 1078, 378, 27, 206, 1, 8280, 2, 264, 5, 848, 4, 4303, 244, 3800, 9, 8, 254, 24, 1, 3865, 6, 7481, 454, 177, 27, 512, 360, 14, 9004, 1185, 6, 3097, 5, 177, 27, 114, 81, 4, 2911, 3545, 366, 4, 18781, 3, 7481, 7335, 6, 4186, 2159, 29, 1, 876, 195, 32, 1419, 29, 1427, 776, 2, 2276, 776, 8, 17, 178, 2, 42, 9, 1, 8760, 3, 3800, 3556, 8, 35, 139, 11, 13, 34, 252, 1, 2664, 2, 3800, 164, 51, 5339, 11, 24, 33, 180, 8, 9, 11, 768, 3800, 901, 24, 11, 25, 1, 12305, 3, 193, 2, 231, 7, 103, 4, 2151, 366, 9, 51, 986, 30, 148, 1, 341, 272, 668, 19613, 5, 177, 598, 13975, 6, 5, 289, 98, 5474, 5, 560, 155, 173, 5, 205, 29, 142, 14, 1583, 512, 11, 182, 214, 2, 84, 44, 35, 38, 155, 213, 4, 505, 60, 3389, 325, 32, 944, 6, 3800, 5, 1, 2667, 25, 11, 4093, 3800, 3556, 8, 3355, 405, 155, 1222, 166, 464, 22, 25, 11, 4093, 113, 2015, 216, 4, 5477, 3260, 740, 15, 138, 524, 405, 155, 1222, 166, 464, 7, 84, 11, 25, 1, 2201, 597, 218, 679, 6, 162, 4, 177, 8760, 44, 618, 269, 154, 3324, 27, 19, 169, 1667, 62, 249, 269, 1815, 2123, 246, 22, 50, 3865, 164, 126, 195, 39, 80, 42, 13, 851, 11, 66, 19, 178, 2, 116, 5169, 27, 508, 791, 178, 2, 116, 4, 4303, 27, 690, 216, 4, 4303, 2254, 81, 124, 184, 1293, 6, 71, 693, 500, 82, 27, 934, 18, 350, 27, 247, 1517, 21, 107, 91, 7, 65, 84, 14, 8, 1913, 1, 4293, 788, 24, 66, 65, 155, 2, 1, 468, 7, 84, 14, 1583, 52, 423, 83, 2343, 752, 2777, 12620, 6, 921, 65, 345, 94, 71, 4293, 48, 6, 745, 154, 1, 2505, 2194, 2186, 347, 8760, 2543, 2, 1853, 776, 150, 432, 5, 5600, 7481, 3122, 39, 52, 83, 32, 1317, 1765, 6, 7481, 114, 281, 2777, 2394, 110, 17, 18, 1141, 124, 363, 22, 17, 1, 2505, 19, 4, 8, 18113, 7, 476, 9, 164, 8760, 27, 18, 31, 1, 117, 258, 83, 7356, 7, 86, 476, 9, 52, 206, 4, 1235, 786, 6, 367, 2, 1, 19701, 24, 224, 209, 889, 5, 954, 76, 111, 7, 65, 64, 550, 139, 42, 58, 2012, 3765, 244, 5169, 5, 9, 9155, 25, 86, 1113, 12, 1, 7481, 5, 2394, 207, 24, 21, 7, 145, 67, 1, 2505, 18, 83, 6, 39, 246, 153, 1, 693, 602, 3, 198, 39, 18, 83, 3416, 4708, 31, 1, 3670, 114, 23, 1, 4293, 70, 1442, 7, 84, 6, 1, 238, 3, 4, 3556, 22, 5169, 1, 951, 34, 2151, 21, 1, 586, 2291, 242, 12, 1, 723, 888, 4437, 536, 7, 84, 9, 96, 5928, 9, 1, 366, 41, 3027, 20, 4, 951, 135, 21, 3556, 22, 5169, 495, 111, 27, 122, 66, 244, 77, 184, 24, 2151, 203, 21, 624, 2021, 107, 31, 258, 46, 19, 6, 413, 2597, 146, 14, 88, 17, 122, 180, 55, 68, 1797, 24, 1, 2589, 791, 48, 19, 13, 409, 1760, 52, 340, 81, 4, 1797, 180, 878, 5789, 390, 4, 2040, 3122, 6, 16841, 350, 27, 19, 390, 4, 2040, 3122, 6, 7481, 14, 8, 742, 2, 1, 13460, 3, 12497, 4396, 27, 18, 1, 12812, 259, 1078, 6, 668, 11, 203, 21, 624, 339, 111, 27, 19, 2151, 312, 24, 27, 19, 360, 50, 1268, 82, 192, 195, 7, 84, 1, 150, 490, 3, 2506, 7, 2181, 2, 4, 108, 1543, 1159, 2828, 2, 36, 1, 1102, 66, 2123, 72, 31, 3014, 468, 96, 1120, 2, 4024, 24, 42, 13, 84, 9, 23, 1, 2589, 3333, 1, 2308, 341, 752, 9, 6, 1, 195, 154, 27, 41, 2593, 71, 7481, 2034, 7, 1391, 834, 11, 111, 6, 9, 238, 1, 12305, 3, 32, 8227, 366, 38, 16, 1344, 20, 22, 204, 4396, 7, 139, 630, 1, 333, 3, 1808, 5, 1, 790, 3, 1, 139, 13, 881, 11, 62, 111, 13, 65, 74, 124, 44, 1, 23, 224, 1, 238, 91, 1558, 262, 24, 23, 11, 339, 1, 238, 91, 98, 2116, 2, 695, 53, 2288]
dean j posting from 8 pitt edu writes i wrote when the pens got mario granted there was big publicity etc etc and interest was immediately generated gretzky did the same thing for la however neither team would have seen a marked improvement in attendance if the team record did not improve in the year before lemieux came pittsburgh finished with 38 points following his arrival the pens finished with 53 76 72 81 87 72 88 and 87 points with a couple of stanley cups thrown in it was at this point the pens attendance was near capacity 34 out of 40 yet they hadn't made the playoffs since 1982 how do you explain a 6th place team breaking attendance records when they haven't been to the playoffs in 7 years mario lemieux is the explanation imho you could make a case that the expectation of an improving team that would make the playoffs is the reason funny you should mention it this is exactly the case i was going to make but i think the reason is lemieux had a 168 point season and was the first non gretzky to win the hart and ross since 1980 people turned out to watch him play i will grant that a star like mario will draw fans even if the team sucks but this is short term only i still do not think the attendance increase will last unless the team is a winning competitive improving butt kicking one pittsburgh was still getting better so people continued to support them if they suddenly dropped to say 50 points you'd have knee surgery for some of the people jumping off the also the following year 88 89 the pens had 89 points not 87 ok my numbers came from the nhl guide and record book they made the transaction to try and build a winner around mario that is true but the improvement in attendance came before they started doing this coffey late in 1987 and before they even had a playoff bound team a doubling of attendance occured in 1984 85 from the previous year an increase from 38 points to 53 points is not going to do that the arrival of mario lemieux is what did it you can give the credit to mario since he deserves it but my point is that it wasn't mario himself but it was the expectation of things to come i e a winning team that he created by being the next great hockey superstar and before anybody jumps in and says i'm picking and words go back and read from where this thread started it might help to think about what would go through a mind who suddenly found an interest in mario and the pens was it gee mario lemieux is amazing i'll go watch him play or was it gee now we've got a kick ass guy on our side i'll go watch him play i think it was the latter similar thing happened in l a before arrival about per game after constant they are still selling out every game despite showing little or no improvement since first year there how do you explain it people are going to see gretzky they certainly aren't going to see a winner they haven't got a winner they've had much better teams in their past history than they currently have yet they didn't draw as well then i don't think this is accurate the tickets sell but people don't go to the games i think this thread has already been discussed season ticket holders in la don't always use their tickets so in effect after the kings initial success following arrival 68 to 91 points same source and corresponding attendance jump there has been an effective drop in attendance even though ticket sales may not have changed much whether or not the kings are a is debatable i claim that since arrival they have at the very least been competitive i also claim that has made a stupid move in trying to the dynasty but that's another story and included only because i don't like anyway did do some heavy marketing around gretzky and that undoubtedly was also responsible for the attendance and sales etc but as i said when the kings have been in there little over the past couple of years there have been empty seats at the forum even if the tickets were sold i think in the case of a lemieux or gretzky the player can winning as the major drawing power for the short term imo although i think that it's inevitable that the team will improve with a player such as lemieux or gretzky simply because they make people around them better but winning sure as hell helps well at least we are in full agreement here this does not make point any more valid but the jets aren't so are you saying roger has ever had a valid point couldn't resist getting a huge jump in productivity yet they are getting a huge jump in attendance this is due to the emergence of teemu selanne they have the 17th best record in hockey it sure as hell isn't because they are winning yes but they are doing no worse than last year i think the same type of reasoning i applied to a new pittsburgh fan applies to all the extra people showing up at winnipeg games it's difficult to predict but do you think that if the jets miss the playoffs next season that in the year after they will maintain their attendance levels i seriously doubt it because in that case the expectation of an improving team would be gone with or without selanne i did provide the example of rocket and the toronto of the did you leave it out because you don't know much about the if that's the case then fair enough but if it isn't the case then i'm curious to hear your explanation
Shape of data tensor: (19997, 1000)
Shape of label tensor: (19997, 20)
In [6]:
# split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
num_validation_samples = int(VALIDATION_SPLIT * data.shape[0])

x_train = data[:-num_validation_samples]
y_train = labels[:-num_validation_samples]
x_val = data[-num_validation_samples:]
y_val = labels[-num_validation_samples:]
In [7]:
print('Preparing embedding matrix.')

# prepare embedding matrix
num_words = min(MAX_NB_WORDS, len(word_index))
embedding_matrix = np.zeros((num_words, EMBEDDING_DIM))
for word, i in word_index.items():
    if i >= MAX_NB_WORDS:
        continue
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector

# load pre-trained word embeddings into an Embedding layer
# note that we set trainable = False so as to keep the embeddings fixed
embedding_layer = Embedding(num_words,
                            EMBEDDING_DIM,
                            weights=[embedding_matrix],
                            input_length=MAX_SEQUENCE_LENGTH,
                            trainable=False)

Preparing embedding matrix.
In [8]:
print('Training model.')

# train a 1D convnet with global maxpooling
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(35)(x)
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)

model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])

model.fit(x_train, y_train,
          batch_size=128,
          epochs=10,
          validation_data=(x_val, y_val))

Training model.
Train on 15998 samples, validate on 3999 samples
Epoch 1/10
  128/15998 [..............................] - ETA: 50s - loss: 3.0036 - acc: 0.0547
/home/dadebarr/anaconda3/lib/python3.5/site-packages/cntk/core.py:351: UserWarning: your data is of type "float64", but your input variable (uid "Input176") expects "<class 'numpy.float32'>". Please convert your data beforehand to speed up training.
  (sample.dtype, var.uid, str(var.dtype)))

15998/15998 [==============================] - 15s - loss: 2.4271 - acc: 0.2038 - val_loss: 1.7163 - val_acc: 0.4129
Epoch 2/10
15998/15998 [==============================] - 13s - loss: 1.5241 - acc: 0.4671 - val_loss: 1.4080 - val_acc: 0.5206
Epoch 3/10
15998/15998 [==============================] - 13s - loss: 1.1756 - acc: 0.6001 - val_loss: 1.1393 - val_acc: 0.6022
Epoch 4/10
15998/15998 [==============================] - 13s - loss: 0.9662 - acc: 0.6710 - val_loss: 1.0123 - val_acc: 0.6494
Epoch 5/10
15998/15998 [==============================] - 13s - loss: 0.8207 - acc: 0.7247 - val_loss: 0.9454 - val_acc: 0.6807
Epoch 6/10
15998/15998 [==============================] - 13s - loss: 0.7019 - acc: 0.7638 - val_loss: 0.8648 - val_acc: 0.7082
Epoch 7/10
15998/15998 [==============================] - 13s - loss: 0.6034 - acc: 0.7946 - val_loss: 0.9057 - val_acc: 0.6964
Epoch 8/10
15998/15998 [==============================] - 13s - loss: 0.5154 - acc: 0.8234 - val_loss: 0.9505 - val_acc: 0.7089
Epoch 9/10
15998/15998 [==============================] - 13s - loss: 0.4319 - acc: 0.8543 - val_loss: 0.9873 - val_acc: 0.7047
Epoch 10/10
15998/15998 [==============================] - 13s - loss: 0.3735 - acc: 0.8729 - val_loss: 0.8736 - val_acc: 0.7307
Out[8]:
<keras.callbacks.History at 0x7f2302dbe710>
In [ ]: