Tensorflow从0到1(4)之神经网络

一维数据集上的神经网络

# 1 引入包，创建会话
import tensorflow as tf
import numpy as np
sess = tf.Session()

# 2 初始化数据
data_size = 25
data_1d = np.random.normal(size=data_size)
x_input_1d = tf.placeholder(dtype=tf.float32, shape=[data_size])

# 3 定义卷积层
def conv_layer_1d(input_1d, my_filter):
    # Make 1d input 4d
    input_2d = tf.expand_dims(input_1d, 0)
    input_3d = tf.expand_dims(input_2d, 0)
    input_4d = tf.expand_dims(input_3d, 3)
    # Perform convolution
    convolution_output = tf.nn.conv2d(input_4d, filter=my_filter,
                                      strides=[1,1,1,1], padding='VALID')
    conv_output_1d = tf.squeeze(convolution_output)
    return conv_output_1d
    # Now drop extra dimensions

my_filter = tf.Variable(tf.random_normal(shape=[1,5,1,1]))
my_convolution_output = conv_layer_1d(x_input_1d, my_filter)

# 4 激励函数
def activation(input_1d):
    return tf.nn.relu(input_1d)
my_activation_output = activation(my_convolution_output)

# 池化
def max_pool(input_1d, width):
    # First we make the 1d input into 4d.
    input_2d = tf.expand_dims(input_1d, 0)
    input_3d = tf.expand_dims(input_2d, 0)
    input_4d = tf.expand_dims(input_3d, 3)
    # Perform the max pool operation
    pool_output = tf.nn.max_pool(input_4d, ksize=[1, 1, width, 1], strides=[1, 1, 1, 1],
                                 padding='VALID')
    pool_output_1d = tf.squeeze(pool_output)
    return pool_output_1d
my_maxpool_output = max_pool(my_activation_output, width=5)

# 全连接层
def fully_connected(input_layer, num_outputs):
    # Create weights
    weight_shape = tf.squeeze(tf.stack([tf.shape(input_layer), [num_outputs]]))
    weight = tf.random_normal(weight_shape, stddev=0.1)
    bias = tf.random_normal(shape=[num_outputs])
    # make input into 2d
    input_layer_2d = tf.expand_dims(input_layer, 0)
    # perform fully connected operations
    full_output = tf.add(tf.matmul(input_layer_2d, weight), bias)
    # Drop extra dimmensions
    full_output_1d = tf.squeeze(full_output)
    return full_output_1d

my_full_output = fully_connected(my_maxpool_output, 5)

# 初始化变量，运行计算图大阴每层输出结果
init = tf.global_variables_initializer()
sess.run(init)
feed_dict = {x_input_1d:data_1d}
# Convolution Output
print("Input = array of length 25")
print("Convolution w/filter length = 5, stride size = 1, results in an array of legth 21:")
print(sess.run(my_convolution_output, feed_dict = feed_dict))
# Activation Output
print('\nInput = the above array of length 21')
print('Relu element wise returns the array of length 21:')
print(sess.run(my_activation_output, feed_dict=feed_dict))
# Maxpool output
print('\nInput = the above array of length 21')
print('MaxPool, window length = 5, stride size = 1, results in the array of length 17')
print(sess.run(my_maxpool_output, feed_dict=feed_dict))
# Fully Connected Output
print('Input = the above array of length 17')
print('Fully connected layer on all four rows with five outputs')
print(sess.run(my_full_output, feed_dict=feed_dict))
# 关闭会话
sess.close()

输出结果如下：

Input = array of length 25
Convolution w/filter length = 5, stride size = 1, results in an array of legth 21:
[ 0.7306204   0.09220226 -0.8647339  -1.7677759   3.0679996  -0.42977548
 -1.4834487   2.084762   -0.63769084 -1.6181873   0.8859257   0.94589835
 -2.3447719   1.4659762   0.86647564 -0.5625909   0.02268941  1.3069543
 -1.5059514   3.0157318  -2.7027912 ]

Input = the above array of length 21
Relu element wise returns the array of length 21:
[0.7306204  0.09220226 0.         0.         3.0679996  0.
 0.         2.084762   0.         0.         0.8859257  0.94589835
 0.         1.4659762  0.86647564 0.         0.02268941 1.3069543
 0.         3.0157318  0.        ]

Input = the above array of length 21
MaxPool, window length = 5, stride size = 1, results in the array of length 17
[3.0679996  3.0679996  3.0679996  3.0679996  3.0679996  2.084762
 2.084762   2.084762   0.94589835 1.4659762  1.4659762  1.4659762
 1.4659762  1.4659762  1.3069543  3.0157318  3.0157318 ]
Input = the above array of length 17
Fully connected layer on all four rows with five outputs
[ 1.8550391  -1.1319994   0.44229037 -1.3700286  -1.7920521 ]

卷积层

首先，卷积层输入序列是25个元素的一维数组。卷积层的功能是相邻5个元素与过滤器（长度为5的向量）内积。因为移动步长为1，所以25个元素的序列中一共有21个相邻为5的序列，最终输出也是5。

激励函数

将卷积成的输出，21个元素的向量通过relu函数逐元素转化。输出仍是21个元素的向量。

池化层，最大值池化

取相邻5个元素的最大值。输入21个元素的序列，输出17个元素的序列。

全连接层

上述17个元素通过全连接层，有5个输出。
注意上述过程的输出都做了维度的裁剪。但在每一步的过程中都是扩充成4维张量操作的。

二维数据上的神经网络

# 1 引入包，创建会话
import tensorflow as tf
import numpy as np
sess = tf.Session()

# 2 创建数据和占位符
data_size = [10, 10]
data_2d = np.random.normal(size=data_size)
x_input_2d = tf.placeholder(dtype=tf.float32, shape=data_size)

# 3 卷积层：2x2过滤器
def conv_layer_2d(input_2d, my_filter):
    # First, change 2d input to 4d
    input_3d = tf.expand_dims(input_2d, 0)
    input_4d = tf.expand_dims(input_3d, 3)
    # Perform convolution
    convolution_output = tf.nn.conv2d(input_4d, filter=my_filter, strides=[1,2,2,1], padding='VALID')
    # Drop extra dimensions
    conv_output_2d = tf.squeeze(convolution_output)
    return conv_output_2d
my_filter = tf.Variable(tf.random_normal(shape=[2,2,1,1]))
my_convolution_output = conv_layer_2d(x_input_2d, my_filter)
# 4 激励函数
def activation(input_2d):
    return tf.nn.relu(input_2d)
my_activation_output = activation(my_convolution_output)
# 5 池化层
def max_pool(input_2d, width, height):
    # Make 2d input into 4d
    input_3d = tf.expand_dims(input_2d, 0)
    input_4d = tf.expand_dims(input_3d, 3)
    # Perform max pool
    pool_output = tf.nn.max_pool(input_4d, ksize=[1, height, width, 1], strides=[1,1,1,1], padding='VALID')
    # Drop extra dimensions
    pool_output_2d = tf.squeeze(pool_output)
    return pool_output_2d
my_maxpool_output = max_pool(my_activation_output, width=2, height=2)
# 6 全连接层
def fully_connected(input_layer, num_outputs):
    # Flatten into 1d
    flat_input = tf.reshape(input_layer, [-1])
    # Create weights
    weight_shape = tf.squeeze(tf.stack([tf.shape(flat_input), [num_outputs]]))
    weight = tf.random_normal(weight_shape, stddev=0.1)
    bias = tf.random_normal(shape=[num_outputs])
    # Change into 2d
    input_2d = tf.expand_dims(flat_input, 0)
    # Perform fully connected operations
    full_output = tf.add(tf.matmul(input_2d, weight), bias)
    # Drop extra dimensions
    full_output_2d = tf.squeeze(full_output)
    return full_output_2d
my_full_output = fully_connected(my_maxpool_output, 5)
# 7 初始化变量
init = tf.initialize_all_variables()
sess.run(init)

feed_dict = {x_input_2d: data_2d}
# 8 打印每层输出结果
# Convolution Output
print('Input = [10 x 10] array')
print('2x2 Convolution, stride size = [2x2], results in the [5x5] array:')
print(sess.run(my_convolution_output, feed_dict=feed_dict))
# Activation Output
print('\nInput = the above [5x5] array')
print('Relu element wise returns the [5x5] array:')
print(sess.run(my_activation_output, feed_dict=feed_dict))
# Max Pool Output
print('\nInput = the above [5x5] array')
print('[2x2] MaxPool, stride size = [1x1] results in the [4x4] array:')
print(sess.run(my_maxpool_output, feed_dict = feed_dict))
# Fully connected output
print('\nInput = the above [4x4] array')
print('Fully connected layer on all four rows with five outputs:')
print(sess.run(my_full_output, feed_dict=feed_dict))

输出结果如下：

Input = [10 x 10] array
2x2 Convolution, stride size = [2x2], results in the [5x5] array:
[[ 0.80993664 -1.2700474   0.27375805  0.54493535 -1.0037322 ]
 [-1.2054954   2.7807589  -0.9015032  -0.24516574  2.126141  ]
 [ 0.19843565 -0.3517378   2.624067   -3.2827137   1.0169035 ]
 [-1.3321284  -0.98290706  0.7477172   1.655221    1.5588429 ]
 [ 1.2763401   0.88586557 -2.230918   -1.5759512   1.1120629 ]]

Input = the above [5x5] array
Relu element wise returns the [5x5] array:
[[0.80993664 0.         0.27375805 0.54493535 0.        ]
 [0.         2.7807589  0.         0.         2.126141  ]
 [0.19843565 0.         2.624067   0.         1.0169035 ]
 [0.         0.         0.7477172  1.655221   1.5588429 ]
 [1.2763401  0.88586557 0.         0.         1.1120629 ]]

Input = the above [5x5] array
[2x2] MaxPool, stride size = [1x1] results in the [4x4] array:
[[2.7807589  2.7807589  0.54493535 2.126141  ]
 [2.7807589  2.7807589  2.624067   2.126141  ]
 [0.19843565 2.624067   2.624067   1.655221  ]
 [1.2763401  0.88586557 1.655221   1.655221  ]]

Input = the above [4x4] array
Fully connected layer on all four rows with five outputs:
[ 0.7709798  -0.2126801  -0.7047844   0.89408153 -0.46939346]

TensorFlow 实现多层神经网络

# 1 引入包
import tensorflow as tf
import matplotlib.pyplot as plt
import requests
import numpy as np
import os
import csv
sess = tf.Session()

# 2 导入数据
# name of data file
birth_weight_file = 'birth_weight.csv'
birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master' \
                '/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'

# Download data and create data file if file does not exist in current directory
if not os.path.exists(birth_weight_file):
    birth_file = requests.get(birthdata_url)
    birth_data = birth_file.text.split('\r\n')
    birth_header = birth_data[0].split('\t')
    birth_data = [[float(x) for x in y.split('\t') if len(x) >= 1]
                  for y in birth_data[1:] if len(y) >= 1]
    with open(birth_weight_file, "w") as f:
        writer = csv.writer(f)
        writer.writerows([birth_header])
        writer.writerows(birth_data)

# read birth weight data into memory
birth_data = []
with open(birth_weight_file, newline='') as csvfile:
    csv_reader = csv.reader(csvfile)
    birth_header = next(csv_reader)
    for row in csv_reader:
        if len(row)>0:
            birth_data.append(row)

birth_data = [[float(x) for x in row] for row in birth_data]

# Extract y-target (birth weight)
y_vals = np.array([x[8] for x in birth_data])

# Filter for features of interest
cols_of_interest = ['AGE', 'LWT', 'RACE', 'SMOKE', 'PTL', 'HT', 'UI']
x_vals = np.array([[x[ix] for ix, feature in enumerate(birth_header) if feature in cols_of_interest]
                   for x in birth_data])

# 3 设置种子
seed = 4
tf.set_random_seed(seed)
np.random.seed(seed)
batch_size = 100

# 4 划分训练集和测试集
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False)
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices)))
x_vals_train = x_vals[train_indices]
x_vals_test = x_vals[test_indices]
y_vals_train = y_vals[train_indices]
y_vals_test = y_vals[test_indices]

def normalize_cols(m):
    col_max = m.max(axis=0)
    col_min = m.min(axis=0)
    return (m - col_min) /(col_max - col_min)

x_vals_train = np.nan_to_num(normalize_cols(x_vals_train))
x_vals_test = np.nan_to_num(normalize_cols(x_vals_test))

# 5 定义一个设置变量和bias的函数
def init_weight(shape, st_dev):
    weight = tf.Variable(tf.random_normal(shape, stddev=st_dev))
    return weight
def init_bias(shape, st_dev):
    bias = tf.Variable(tf.random_normal(shape, stddev=st_dev))
    return bias

# 6 初始化占位符
x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32)
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

# 7 创建全连接层函数，方便重复使用
def fully_connected(input_layer, weights, biases):
    layer = tf.add(tf.matmul(input_layer, weights), biases)
    return tf.nn.relu(layer)

# 8 创建算法模型
# Create second layer (25 hidden nodes)
weight_1 = init_weight(shape=[7, 25], st_dev=10.0)
bias_1 = init_weight(shape=[25], st_dev=10.0)
layer_1 = fully_connected(x_data, weight_1, bias_1)

# Create second layer (10 hidden nodes)
weight_2 = init_weight(shape=[25, 10], st_dev=10.0)
bias_2 = init_weight(shape=[10], st_dev=10.0)
layer_2 = fully_connected(layer_1, weight_2, bias_2)

# Create third layer (3 hidden nodes)
weight_3 = init_weight(shape=[10, 3], st_dev=10.0)
bias_3 = init_weight(shape=[3], st_dev=10.0)
layer_3 = fully_connected(layer_2, weight_3, bias_3)

# Create output layer (1 output value)
weight_4 = init_weight(shape=[3, 1], st_dev=10.0)
bias_4 = init_bias(shape=[1], st_dev=10.0)
final_output = fully_connected(layer_3, weight_4, bias_4)

# 9 L1损失函数
loss = tf.reduce_mean(tf.abs(y_target - final_output))
my_opt = tf.train.AdamOptimizer(0.05)
train_step = my_opt.minimize(loss)
init = tf.global_variables_initializer()
sess.run(init)

# 10 迭代200
# Initialize the loss vectors
loss_vec = []
test_loss = []
for i in range(200):
    # Choose random indices for batch selection
    rand_index = np.random.choice(len(x_vals_train), size=batch_size)
    # Get random batch
    rand_x = x_vals_train[rand_index]
    rand_y = np.transpose([y_vals_train[rand_index]])
    # Run the training step
    sess.run(train_step, feed_dict={x_data: rand_x, y_target:rand_y})
    # Get and store the train loss
    temp_loss = sess.run(loss, feed_dict = {x_data:rand_x, y_target:rand_y})
    loss_vec.append(temp_loss)
    # get and store the test loss
    test_temp_loss = sess.run(loss, feed_dict = {x_data:x_vals_test, y_target:np.transpose([y_vals_test])})
    test_loss.append(test_temp_loss)
    if (i+1)% 25 == 0:
        print('Generation: ' + str(i+1)+'.Loss = ' + str(temp_loss))

# 12 绘图
plt.plot(loss_vec, 'k-', label='Train Loss')
plt.plot(test_loss, 'r--', label='Test Loss')
plt.title('Loss per Generation')
plt.xlabel('Generation')
plt.ylabel('Loss')
plt.legend(loc='upper right')
plt.show()

# Model Accuracy
actuals = np.array([x[0] for x in birth_data])
test_actuals = actuals[test_indices]
train_actuals = actuals[train_indices]
test_preds = [x[0] for x in sess.run(final_output, feed_dict={x_data: x_vals_test})]
train_preds = [x[0] for x in sess.run(final_output, feed_dict={x_data: x_vals_train})]
test_preds = np.array([0.0 if x < 2500.0 else 1.0 for x in test_preds])
train_preds = np.array([0.0 if x < 2500.0 else 1.0 for x in train_preds])
# Print out accuracies
test_acc = np.mean([x == y for x, y in zip(test_preds, test_actuals)])
train_acc = np.mean([x == y for x, y in zip(train_preds, train_actuals)])
print('On predicting the category of low birthweight from regression output (<2500g):')
print('Test Accuracy: {}'.format(test_acc))
print('Train Accuracy: {}'.format(train_acc))

实现了一个含有三层隐藏层的全连接神经网络。

---

线性预测模型的优化

# 1 导入必要的库
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
import requests
import os
import csv
sess = tf.Session()

# 加载数据集，进行数据抽取和归一化
# name of data file
birth_weight_file = 'birth_weight.csv'
birthdata_url = 'https://github.com/nfmcclure/tensorflow_cookbook/raw/master' \
                '/01_Introduction/07_Working_with_Data_Sources/birthweight_data/birthweight.dat'

# Download data and create data file if file does not exist in current directory
if not os.path.exists(birth_weight_file):
    birth_file = requests.get(birthdata_url)
    birth_data = birth_file.text.split('\r\n')
    birth_header = birth_data[0].split('\t')
    birth_data = [[float(x) for x in y.split('\t') if len(x) >= 1]
                  for y in birth_data[1:] if len(y) >= 1]
    with open(birth_weight_file, "w") as f:
        writer = csv.writer(f)
        writer.writerows([birth_header])
        writer.writerows(birth_data)

# read birth weight data into memory
birth_data = []
with open(birth_weight_file, newline='') as csvfile:
    csv_reader = csv.reader(csvfile)
    birth_header = next(csv_reader)
    for row in csv_reader:
        if len(row)>0:
            birth_data.append(row)

birth_data = [[float(x) for x in row] for row in birth_data]

# Extract y-target (birth weight)
y_vals = np.array([x[0] for x in birth_data])

# Filter for features of interest
cols_of_interest = ['AGE', 'LWT', 'RACE', 'SMOKE', 'PTL', 'HT', 'UI']
x_vals = np.array([[x[ix] for ix, feature in enumerate(birth_header) if feature in cols_of_interest]
                   for x in birth_data])

# 4 划分训练集和测试集
train_indices = np.random.choice(len(x_vals), round(len(x_vals)*0.8), replace=False)
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices)))
x_vals_train = x_vals[train_indices]
x_vals_test = x_vals[test_indices]
y_vals_train = y_vals[train_indices]
y_vals_test = y_vals[test_indices]

def normalize_cols(m):
    col_max = m.max(axis=0)
    col_min = m.min(axis=0)
    return (m - col_min) /(col_max - col_min)

x_vals_train = np.nan_to_num(normalize_cols(x_vals_train))
x_vals_test = np.nan_to_num(normalize_cols(x_vals_test))

# 3 声明批量大小和占位符
batch_size = 90
x_data = tf.placeholder(shape=[None, 7], dtype=tf.float32)
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

# 4 声明函数来初始化变量和层
def init_variable(shape):
    return tf.Variable(tf.random_normal(shape=shape))

# Create a logistic layer definition
def logistic(input_layer, multiplication_weight, bias_weight, activation=True):
    linear_layer = tf.add(tf.matmul(input_layer, multiplication_weight), bias_weight)

    if activation :
        return tf.nn.sigmoid(linear_layer)
    else :
        return linear_layer

# 5 声明神经网络的两个隐藏层和输出层
# First logistic layer (7 inputs to 14 hidden nodes)
A1 = init_variable(shape=[7, 14])
b1 = init_variable(shape=[14])
logistic_layer1 = logistic(x_data, A1, b1)
# Second logistic layer (14 inputs to 5 hidden nodes)
A2 = init_variable(shape=[14, 5])
b2 = init_variable(shape=[5])
logistic_layer2 = logistic(logistic_layer1, A2, b2)
# Final output layer (5 hidden nodes to 1 output)
A3 = init_variable(shape=[5, 1])
b3 = init_variable(shape=[1])
final_output = logistic(logistic_layer2, A3, b3, activation=False)

# 6 声明损失函数和优化方法
# Create loss function
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
        labels = y_target, logits=final_output))
# Declare optimizer
my_opt = tf.train.AdamOptimizer(learning_rate = 0.002)
train_step = my_opt.minimize(loss)
# Initialize variables
init = tf.global_variables_initializer()
sess.run(init)

# 7 评估精度
prediction = tf.round(tf.nn.sigmoid(final_output))
predictions_correct = tf.cast(tf.equal(prediction, y_target), tf.float32)
accuracy = tf.reduce_mean(predictions_correct)

# 8 迭代训练模型
# Initialize loss and accuracy vectors
loss_vec = []
train_acc = []
test_acc = []
for i in range(1500):
    # Select random indicies for batch selection
    rand_index = np.random.choice(len(x_vals_train), size=batch_size)
    # Select batch
    rand_x = x_vals_train[rand_index]
    rand_y = np.transpose([y_vals_train[rand_index]])
    # Run training step
    sess.run(train_step, feed_dict={x_data:rand_x, y_target:rand_y})
    # Get training loss
    temp_loss = sess.run(loss, feed_dict={x_data:rand_x, y_target:rand_y})
    loss_vec.append(temp_loss)
    # Get training accuracy
    temp_acc_train = sess.run(accuracy, feed_dict = {x_data:x_vals_train, y_target:np.transpose([y_vals_train])})
    train_acc.append(temp_acc_train)
    # Get test accuracy
    temp_acc_test = sess.run(accuracy, feed_dict = {x_data:x_vals_test, y_target:np.transpose([y_vals_test])})
    test_acc.append(temp_acc_test)
    if (i+1)%150==0:
        print('Loss = ' + str(temp_loss))

# 9 绘图
# Plot loss over time
plt.plot(loss_vec, 'k-')
plt.title('Cross Entropy Loss per Generation')
plt.xlabel('Generation')
plt.ylabel('Cross Entropy Loss')
plt.show()
# Plot train and test accuracy
plt.plot(train_acc, 'k-', label='Train Set Accuracy')
plt.plot(test_acc, 'r--', label='Test Set Accuracy')
plt.title('Train and Test Accuracy')
plt.xlabel('Generation')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
plt.show()

这一个仍然是全连接，只是只有两层隐藏层，节点数也减少了。