20191130

发表于 | 分类于

思考题

在scope例子中加入tensorboard, 并观察可视化图与没有scope的变化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import tensorflow as tf

with tf.device('/cpu:0'):
    with tf.variable_scope('foo'):
        x_init1 = tf.get_variable('init_x', [10], dtype=tf.float32, initializer=tf.random_normal_initializer())[0]
        x = tf.Variable(initial_value=x_init1, name='x')
        y = tf.placeholder(dtype=tf.float32, name='y')
        z = x + y

    with tf.variable_scope('bar'):
        a = tf.constant(3.0) + 4.0

    w = z * a

# 开始利用tf.summary记录图的信息, 需要展示的信息

tf.summary.scalar('scalar_x_init1', x_init1)
tf.summary.scalar(name='scalar_x', tensor=x)
tf.summary.scalar(name='scalar_y', tensor=y)
tf.summary.scalar(name='scalar_z', tensor=z)
tf.summary.scalar(name='scalar_w', tensor=w)

# update操作, 这里只能更新x
assign_op = tf.assign(x, x + 1)

with tf.control_dependencies([assign_op]):
    with tf.device('/gpu:0'):
        out = x * y
        tf.summary.scalar(name='scalar_out', tensor=out)

with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess:
    # merge所有的summary, 触发所有的输出操作
    merged_summary = tf.summary.merge_all()
    # 得到文件的输出对象
    writer = tf.summary.FileWriter('./result', sess.graph)
    # 初始化
    sess.run(tf.global_variables_initializer())
    # print

    for i in range(1, 5):
        summary, r_out, r_x, r_w = sess.run([merged_summary, out, x, w], feed_dict={y: i})
        writer.add_summary(summary, i)
        print("{}, {}, {}".format(r_out, r_x, r_w))
1
tensorboard --logdir tensorflow_excise/result

MNIST DNN中single-layer.py和single-layer-optimization.py的结果是有差距, 寻找原因

编程题

MNIST DNN例子中实现cross entropy;

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# by cangye@hotmail.com
# 引入库
from tensorflow.examples.tutorials.mnist import input_data
import tensorflow as tf

# 获取数据
mnist = input_data.read_data_sets("../data/", one_hot=True)
# 定义全链接层函数
def full_layer(input_tensor, out_dim, name='full'):
    with tf.variable_scope(name):
        shape = input_tensor.get_shape().as_list()
        W = tf.get_variable('W', (shape[1], out_dim), dtype=tf.float32,
                            initializer=tf.truncated_normal_initializer(stddev=0.1))
        b = tf.get_variable('b', [out_dim], dtype=tf.float32, initializer=tf.constant_initializer(0))
        out = tf.matmul(input_tensor, W) + b
    return tf.nn.sigmoid(out)


def model(net, out_dim):
    net = full_layer(net, out_dim, "full_layer1")
    return net


# 定义输入
with tf.variable_scope("inputs"):
    x = tf.placeholder(tf.float32, [None, 784])
    label = tf.placeholder(tf.float32, [None, 10])
# 引入模型
y = model(x, 10)
# 定义损失函数
# loss = tf.reduce_mean(tf.square(y - label))
ce = tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=y)
loss = tf.reduce_mean(ce)
# 用梯度迭代算法
train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
# 用于验证
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(label, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# 定义会话
sess = tf.Session()
# 初始化所有变量
sess.run(tf.global_variables_initializer())
# 迭代过程
train_writer = tf.summary.FileWriter("mnist-logdir", sess.graph)
for itr in range(10000):
    batch_xs, batch_ys = mnist.train.next_batch(100)
    sess.run(train_step, feed_dict={x: batch_xs, label: batch_ys})
    if itr % 10 == 0:
        print("step:%6d  accuracy:" % itr, sess.run(accuracy, feed_dict={x: mnist.test.images,
                                                                         label: mnist.test.labels}))

# 这下面的部分用于绘图
import matplotlib.pyplot as plt
import numpy as np
import matplotlib as mpl

mpl.style.use('fivethirtyeight')
# 获取W取值
with tf.variable_scope("full_layer1", reuse=True):
    W = tf.get_variable("W")
W = sess.run(W.value())
# 绘图过程
fig = plt.figure()
ax = fig.add_subplot(221)
ax.matshow(np.reshape(W[:, 1], [28, 28]), cmap=plt.get_cmap("Purples"))
ax = fig.add_subplot(222)
ax.matshow(np.reshape(W[:, 2], [28, 28]), cmap=plt.get_cmap("Purples"))
ax = fig.add_subplot(223)
ax.matshow(np.reshape(W[:, 3], [28, 28]), cmap=plt.get_cmap("Purples"))
ax = fig.add_subplot(224)
ax.matshow(np.reshape(W[:, 4], [28, 28]), cmap=plt.get_cmap("Purples"))
plt.show()

在IRIS例子中实现cross entropy代替MSE和sigmoid

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# by cangye@hotmail.com
# TensorFlow入门实例

import pandas as pd
import tensorflow as tf
import tensorflow.contrib.slim as slim
import numpy as np


def variable_summaries(var, name="layer"):
    with tf.variable_scope(name):
        mean = tf.reduce_mean(var)
        tf.summary.scalar('mean', mean)
        with tf.name_scope('stddev'):
            stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))
        tf.summary.scalar('stddev', stddev)
        tf.summary.scalar('max', tf.reduce_max(var))
        tf.summary.scalar('min', tf.reduce_min(var))
        tf.summary.histogram('histogram', var)


data = pd.read_csv("../data/iris.data.csv")
c_name = set(data.name.values)
print(c_name)
iris_label = np.zeros([len(data.name.values), len(c_name)])
iris_data = data.values[:, :-1]
iris_data = iris_data - np.mean(iris_data, axis=0)
iris_data = iris_data / np.max(iris_data, axis=0)
train_data = []
train_data_label = []
test_data = []
test_data_label = []
for idx, itr_name in enumerate(c_name):
    datas_t = iris_data[data.name.values == itr_name, :]
    labels_t = np.zeros([len(datas_t), len(c_name)])
    labels_t[:, idx] = 1
    train_data.append(datas_t[:30])
    train_data_label.append(labels_t[:30])
    test_data.append(datas_t[30:])
    test_data_label.append(labels_t[30:])
train_data = np.concatenate(train_data)
train_data_label = np.concatenate(train_data_label)
test_data = np.concatenate(test_data)
test_data_label = np.concatenate(test_data_label)
x = tf.placeholder(tf.float32, [None, 4], name="input_x")
label = tf.placeholder(tf.float32, [None, 3], name="input_y")
# 对于sigmoid激活函数而言，效果可能并不理想
net = slim.fully_connected(x, 4, activation_fn=tf.nn.relu, scope='full1', reuse=False)
net = tf.contrib.layers.batch_norm(net)
net = slim.fully_connected(net, 8, activation_fn=tf.nn.relu, scope='full2', reuse=False)
net = tf.contrib.layers.batch_norm(net)
net = slim.fully_connected(net, 8, activation_fn=tf.nn.relu, scope='full3', reuse=False)
net = tf.contrib.layers.batch_norm(net)
net = slim.fully_connected(net, 4, activation_fn=tf.nn.relu, scope='full4', reuse=False)
net = tf.contrib.layers.batch_norm(net)
y = slim.fully_connected(net, 3, activation_fn=tf.nn.sigmoid, scope='full5', reuse=False)

ce = tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=y)
loss = tf.reduce_mean(ce)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(label, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
optimizer = tf.train.GradientDescentOptimizer(0.5)
var_list_w = [var for var in tf.trainable_variables() if "w" in var.name]
var_list_b = [var for var in tf.trainable_variables() if "b" in var.name]
gradient_w = optimizer.compute_gradients(loss, var_list=var_list_w)
gradient_b = optimizer.compute_gradients(loss, var_list=var_list_b)
for idx, itr_g in enumerate(gradient_w):
    variable_summaries(itr_g[0], "layer%d-w-grad" % idx)
for idx, itr_g in enumerate(gradient_b):
    variable_summaries(itr_g[0], "layer%d-b-grad" % idx)
for idx, itr_g in enumerate(var_list_w):
    variable_summaries(itr_g, "layer%d-w" % idx)
for idx, itr_g in enumerate(var_list_b):
    variable_summaries(itr_g, "layer%d-b" % idx)
train_step = optimizer.apply_gradients(gradient_w + gradient_b)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter("logdir-bn", sess.graph)
merged = tf.summary.merge_all()
for itr in range(600):
    sess.run(train_step, feed_dict={x: train_data, label: train_data_label})
    if itr % 30 == 0:
        acc1 = sess.run(accuracy, feed_dict={x: train_data, label: train_data_label})
        acc2 = sess.run(accuracy, feed_dict={x: test_data, label: test_data_label})
        print("step:{:6d}  train:{:.3f} test:{:.3f}".format(itr, acc1, acc2))
        summary = sess.run(merged, feed_dict={x: train_data, label: train_data_label})
        train_writer.add_summary(summary, itr)