从锅炉工到AI专家(9)

#!/usr/bin/env python
# -*- coding=UTF-8 -*-

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Basic word2vec example."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import math
import os
import random
from tempfile import gettempdir
import zipfile

import numpy as np
from six.moves import urllib
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

# Step 1: Download the data.
url = 'http://mattmahoney.net/dc/'

# 从上面的URL下载给定的语料库
# 为了提高速度，这里手工下载后屏蔽了本函数，防止每次运行都重复下载速度太慢
# pylint: disable=redefined-outer-name
def maybe_download(filename, expected_bytes):
  """Download a file if not present, and make sure it's the right size."""
  local_filename = os.path.join(gettempdir(), filename)
  if not os.path.exists(local_filename):
    local_filename, _ = urllib.request.urlretrieve(url + filename,
                                                   local_filename)
  statinfo = os.stat(local_filename)
  if statinfo.st_size == expected_bytes:
    print('Found and verified', filename)
  else:
    print(statinfo.st_size)
    raise Exception('Failed to verify ' + local_filename +
                    '. Can you get to it with a browser?')
  return local_filename

#单词数据包实际下载路径：http://mattmahoney.net/dc/text8.zip
#在这里下载后放到当前目录，所以下面filename做了修改，并且不再调用maybe_download函数
#filename = maybe_download('text8.zip', 31344016)
filename = "./text8.zip"

#从zip包中第一个文件读取所有的数据（实际只有一个文本文件），
#所有的数据只有词，以空格分割，没有标点符号。
#单词之间有语序关系，意思是某文章去掉标点符号之后，每句话中单词的语序仍然存在。
#为了加深印象，可以解压缩text8.zip包，然后显示文本文件看一下，
#文件很大，建议使用more只查看一部分
# Read the data into a list of strings.
def read_data(filename):
  """Extract the first file enclosed in a zip file as a list of words."""
  with zipfile.ZipFile(filename) as f:
    data = tf.compat.as_str(f.read(f.namelist()[0])).split()
  return data

#读取所有单词到字符串数组
vocabulary = read_data(filename)
print('Data size', len(vocabulary))

#整体数据，按照这个下载包是17005207个单词，下面50000是为了演示速度，限制了有效词数
# Step 2: Build the dictionary and replace rare words with UNK token.
vocabulary_size = 50000

def build_dataset(words, n_words):
  """Process raw inputs into a dataset."""
  count = [['UNK', -1]]
  #按照相同词统计数，进行排序，常用词在前面,最前面当然是UNK
  #其次是the/of/and/one，排序靠前的5个单词后面会显示出来...
  count.extend(collections.Counter(words).most_common(n_words - 1))
  dictionary = dict()
  for word, _ in count:
    #计数增加，用于给每个词编一个唯一数字代码
    #UNK是第0个，编码是0，因为加入第一个UNK的时候，dictionary是空，所以len是0。
    #输出看的时候，因为是以单词顺序列出来，所以看着顺序混乱，
    #实际看反查表因为数字在前，看起来会更明显。
    dictionary[word] = len(dictionary) 
  data = list()
  #data最终是数字化之后的words，也就是数字化之后的原文，
  #其中按照原文顺序，每个元素，是该单词的数字编码
  #数字编码是从dictionary中查表找到的，也就是本函数前面数字化单词的过程得到的
  unk_count = 0
  for word in words:
    index = dictionary.get(word, 0)
    if index == 0:  # dictionary['UNK']
      unk_count += 1
    data.append(index)
  count[0][1] = unk_count
  #这个逆转字典是从数字到单词来对应，双向查表用的
  reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
  return data, count, dictionary, reversed_dictionary

# Filling 4 global variables:
# data - list of codes (integers from 0 to vocabulary_size-1).
#   This is the original text but words are replaced by their codes
# count - map of words(strings) to count of occurrences
# dictionary - map of words(strings) to their codes(integers)
# reverse_dictionary - maps codes(integers) to words(strings)
#使用build_dataset函数填充4个全局变量，
#这些全局变量的内容刚才在函数注释中我们都介绍过了
#也可以参考上面官方原本的英文注释
data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,
                                                            vocabulary_size)
#做完了上面的数字化，原文其实就没用了，这里删除以节省内存
del vocabulary  # Hint to reduce memory.
#count表上面说了，是统计出现次数，这里列出最常出现的5个单词
print('Most common words (+UNK)', count[:5])
#数字化后的前10个单词及查表得出的原文,
#注意后面的逆向表查表部分是python特有的语法，其它语言中不多见
print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])

data_index = 0

# Step 3: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
  global data_index
  assert batch_size % num_skips == 0
  assert num_skips <= 2 * skip_window
  batch = np.ndarray(shape=(batch_size), dtype=np.int32)
  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
  span = 2 * skip_window + 1  # [ skip_window target skip_window ]
  buffer = collections.deque(maxlen=span)
  if data_index + span > len(data):
    data_index = 0
  buffer.extend(data[data_index:data_index + span])
  data_index += span
  for i in range(batch_size // num_skips):
    context_words = [w for w in range(span) if w != skip_window]
    words_to_use = random.sample(context_words, num_skips)
    for j, context_word in enumerate(words_to_use):
      batch[i * num_skips + j] = buffer[skip_window]
      labels[i * num_skips + j, 0] = buffer[context_word]
    if data_index == len(data):
      buffer.extend(data[0:span])
      data_index = span
    else:
      buffer.append(data[data_index])
      data_index += 1
  # Backtrack a little bit to avoid skipping words in the end of a batch
  data_index = (data_index + len(data) - span) % len(data)
  return batch, labels

#生成一批用于学习的数据集，这里首先生成一批很小的量
#然后在下面显示出来，用于人为观察生成的数据集是否合理
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
  print(batch[i], reverse_dictionary[batch[i]],
        '->', labels[i, 0], reverse_dictionary[labels[i, 0]])

# Step 4: Build and train a skip-gram model.
#这里定义的常量，才是真正学习的时候生成数据集的尺寸等参数
batch_size = 128
embedding_size = 128  # Dimension of the embedding vector.
# 左右各考虑1个单词
skip_window = 1       # How many words to consider left and right.
# 本窗口完成跳2个单词取样下一个窗口
num_skips = 2         # How many times to reuse an input to generate a label.
num_sampled = 64      # Number of negative examples to sample.

# We pick a random validation set to sample nearest neighbors. Here we limit the
# validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent. These 3 variables are used only for
# displaying model accuracy, they don't affect calculation.
valid_size = 16     # Random set of words to evaluate similarity on.
valid_window = 100  # Only pick dev samples in the head of the distribution.
valid_examples = np.random.choice(valid_window, valid_size, replace=False)


graph = tf.Graph()

with graph.as_default():

  # Input data.
  train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
  train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
  valid_dataset = tf.constant(valid_examples, dtype=tf.int32)

  # Ops and variables pinned to the CPU because of missing GPU implementation
  with tf.device('/cpu:0'):
    # Look up embeddings for inputs.
    embeddings = tf.Variable(
        tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
    embed = tf.nn.embedding_lookup(embeddings, train_inputs)

    # Construct the variables for the NCE loss
    nce_weights = tf.Variable(
        tf.truncated_normal([vocabulary_size, embedding_size],
                            stddev=1.0 / math.sqrt(embedding_size)))
    nce_biases = tf.Variable(tf.zeros([vocabulary_size]))

  # Compute the average NCE loss for the batch.
  # tf.nce_loss automatically draws a new sample of the negative labels each
  # time we evaluate the loss.
  # Explanation of the meaning of NCE loss:
  #   http://mccormickml.com/2016/04/19/word2vec-tutorial-the-skip-gram-model/
  loss = tf.reduce_mean(
      tf.nn.nce_loss(weights=nce_weights,
                     biases=nce_biases,
                     labels=train_labels,
                     inputs=embed,
                     num_sampled=num_sampled,
                     num_classes=vocabulary_size))

  # Construct the SGD optimizer using a learning rate of 1.0.
  optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)

  # Compute the cosine similarity between minibatch examples and all embeddings.
  norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
  normalized_embeddings = embeddings / norm
  valid_embeddings = tf.nn.embedding_lookup(
      normalized_embeddings, valid_dataset)
  similarity = tf.matmul(
      valid_embeddings, normalized_embeddings, transpose_b=True)

  # Add variable initializer.
  init = tf.global_variables_initializer()

# Step 5: Begin training.
num_steps = 100001

with tf.Session(graph=graph) as session:
  # We must initialize all variables before we use them.
  init.run()
  print('Initialized')

  average_loss = 0
  for step in xrange(num_steps):
    batch_inputs, batch_labels = generate_batch(
        batch_size, num_skips, skip_window)
    #可以在tensorflow运算过程中逐批次喂入的数据集是由tf.placeholder定义的，
    #这里把所有要喂入的数据先包装成dict
    feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}

    # We perform one update step by evaluating the optimizer op (including it
    # in the list of returned values for session.run()
    #运行，并逐批次喂入数据
    _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
    average_loss += loss_val

    if step % 2000 == 0:
      if step > 0:
        average_loss /= 2000
      # The average loss is an estimate of the loss over the last 2000 batches.
      #这里显示的是每2000批次平均出来的代价函数返回值
      print('Average loss at step ', step, ': ', average_loss)
      average_loss = 0

    # Note that this is expensive (~20% slowdown if computed every 500 steps)
    if step % 10000 == 0:
      sim = similarity.eval()
      for i in xrange(valid_size):
        valid_word = reverse_dictionary[valid_examples[i]]
        top_k = 8  # number of nearest neighbors
        nearest = (-sim[i, :]).argsort()[1:top_k + 1]
        log_str = 'Nearest to %s:' % valid_word
        for k in xrange(top_k):
          close_word = reverse_dictionary[nearest[k]]
          log_str = '%s %s,' % (log_str, close_word)
        print(log_str)
  final_embeddings = normalized_embeddings.eval()

# Step 6: Visualize the embeddings.


# pylint: disable=missing-docstring
# Function to draw visualization of distance between embeddings.
def plot_with_labels(low_dim_embs, labels, filename):
  assert low_dim_embs.shape[0] >= len(labels), 'More labels than embeddings'
  plt.figure(figsize=(18, 18))  # in inches
  for i, label in enumerate(labels):
    x, y = low_dim_embs[i, :]
    plt.scatter(x, y)
    plt.annotate(label,
                 xy=(x, y),
                 xytext=(5, 2),
                 textcoords='offset points',
                 ha='right',
                 va='bottom')

  plt.savefig(filename)

try:
  # pylint: disable=g-import-not-at-top
  from sklearn.manifold import TSNE
  import matplotlib.pyplot as plt

  tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000, method='exact')
  plot_only = 500
  low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only, :])
  labels = [reverse_dictionary[i] for i in xrange(plot_only)]
  plot_with_labels(low_dim_embs, labels, './tsne.png')

except ImportError as ex:
  print('Please install sklearn, matplotlib, and scipy to show embeddings.')
  print(ex)

#!/usr/bin/env/python
# coding=utf-8
import tensorflow as tf
import numpy as np

input_ids = tf.placeholder(dtype=tf.int32, shape=[None])

#定义一个5x5对角矩阵，样式可以看运行结果第一个输出 
embedding = tf.Variable(np.identity(5, dtype=np.int32))
#使用embedding_lookup检索矩阵，检索数据集是input_ids
input_embedding = tf.nn.embedding_lookup(embedding, input_ids)

sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
print(embedding.eval())
print(sess.run(input_embedding, feed_dict={input_ids:[1, 2, 3, 0, 3, 2, 1]}))

word2vec = tf.load_op_library(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'word2vec_ops.so'))
...
    (words, counts, words_per_epoch, current_epoch, total_words_processed,
     examples, labels) = word2vec.skipgram_word2vec(filename=opts.train_data,
                                                    batch_size=opts.batch_size,
                                                    window_size=opts.window_size,
                                                    min_count=opts.min_count,
                                                    subsample=opts.subsample)

2018-01-16 13:17:14.277603: I word2vec_kernels.cc:200] Data file: /Users/andrew/dev/tensorFlow/word2vec/text8 contains 100000000 bytes, 17005207 words, 253854 unique words, 71290 unique frequent words.
Data file:  /Users/andrew/dev/tensorFlow/word2vec/text8
Vocab size:  71290  + UNK
Words per epoch:  17005207
Eval analogy file:  /Users/andrew/dev/tensorFlow/word2vec/questions-words.txt
Questions:  17827
Skipped:  1717
Epoch    1 Step   150943: lr = 0.024 words/sec =    31527
Eval 1469/17827 accuracy =  8.2%
Epoch    2 Step   301913: lr = 0.023 words/sec =    25120
Eval 2395/17827 accuracy = 13.4%
Epoch    3 Step   452887: lr = 0.021 words/sec =     8842
Eval 3014/17827 accuracy = 16.9%
Epoch    4 Step   603871: lr = 0.020 words/sec =     6615
Eval 3532/17827 accuracy = 19.8%
Epoch    5 Step   754815: lr = 0.019 words/sec =     3007
Eval 3994/17827 accuracy = 22.4%
Epoch    6 Step   905787: lr = 0.018 words/sec =    26590
Eval 4320/17827 accuracy = 24.2%
Epoch    7 Step  1056767: lr = 0.016 words/sec =    35439
Eval 4714/17827 accuracy = 26.4%
Epoch    8 Step  1207755: lr = 0.015 words/sec =      401
Eval 4965/17827 accuracy = 27.9%
Epoch    9 Step  1358735: lr = 0.014 words/sec =    36991
Eval 5276/17827 accuracy = 29.6%
Epoch   10 Step  1509744: lr = 0.013 words/sec =    25069
Eval 5415/17827 accuracy = 30.4%
Epoch   11 Step  1660729: lr = 0.011 words/sec =    28271
Eval 5649/17827 accuracy = 31.7%
Epoch   12 Step  1811667: lr = 0.010 words/sec =    29973
Eval 5880/17827 accuracy = 33.0%
Epoch   13 Step  1962606: lr = 0.009 words/sec =    10225
Eval 6015/17827 accuracy = 33.7%
Epoch   14 Step  2113546: lr = 0.008 words/sec =    21419
Eval 6270/17827 accuracy = 35.2%
Epoch   15 Step  2264489: lr = 0.006 words/sec =    27059
Eval 6434/17827 accuracy = 36.1%

: capital-common-countries
Athens Greece Baghdad Iraq
Athens Greece Bangkok Thailand
Athens Greece Beijing China
Athens Greece Berlin Germany
Athens Greece Bern Switzerland
Athens Greece Cairo Egypt
Athens Greece Canberra Australia
Athens Greece Hanoi Vietnam
Athens Greece Havana Cuba
Athens Greece Helsinki Finland
Athens Greece Islamabad Pakistan
Athens Greece Kabul Afghanistan
Athens Greece London England
Athens Greece Madrid Spain
Athens Greece Moscow Russia
Athens Greece Oslo Norway
Athens Greece Ottawa Canada
Athens Greece Paris France
Athens Greece Rome Italy
Athens Greece Stockholm Sweden
Athens Greece Tehran Iran
Athens Greece Tokyo Japan
Baghdad Iraq Bangkok Thailand
Baghdad Iraq Beijing China
Baghdad Iraq Berlin Germany
Baghdad Iraq Bern Switzerland
...