def _int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=http://www.netofthings.cn/JieJueFangAn/2017-02/[value]))"hljs-function">
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=http://www.netofthings.cn/JieJueFangAn/2017-02/[value]))"hljs-function">
def convert_to(data_set, name): images = data_set.images labels = data_set.labels num_examples = data_set.num_examples
if images.shape[
0] != num_examples:
raise ValueError(
'Images size %d does not match label size %d.' % (images.shape[
0], num_examples)) rows = images.shape[
1] cols = images.shape[
2] depth = images.shape[
3] filename = os.path.join(FLAGS.directory, name +
'.tfrecords') print(
'Writing', filename) writer = tf.python_io.TFRecordWriter(filename)
for index
in range(num_examples): image_raw = images[index].tostring() example = tf.train.Example(features=tf.train.Features(feature={
'height': _int64_feature(rows),
'width': _int64_feature(cols),
'depth': _int64_feature(depth),
'label': _int64_feature(int(labels[index])),
'image_raw': _bytes_feature(image_raw)})) writer.write(example.SerializeToString()) writer.close()
def main(argv): data_sets = mnist.read_data_sets(FLAGS.directory, dtype=tf.uint8, reshape=
False, validation_size=FLAGS.validation_size) convert_to(data_sets.train,
'train') convert_to(data_sets.validation,
'validation') convert_to(data_sets.test,
'test')
if __name__ ==
'__main__': parser = argparse.ArgumentParser() parser.add_argument(
'--directory', type=str, default=
'/tmp/data', help=
'Directory to download data files and write the converted result' ) parser.add_argument(
'--validation_size', type=int, default=
5000, help=
"""\ Number of examples to separate from the training data for the validation set.\ """ ) FLAGS = parser.parse_args() tf.app.run()
二、Tensorflow讀取HDFS數(shù)據(jù)的設(shè)置
文中前面內(nèi)容介紹了HDFS的配置以及將數(shù)據(jù)轉(zhuǎn)換后存儲(chǔ)到HDFS，Tensorflow讀取HDFS時(shí)只需要簡(jiǎn)單的兩步，首先執(zhí)行項(xiàng)目時(shí)需要加入環(huán)境前綴：
CLASSPATH=$($HADOOP_HDFS_HOME/bin/hadoop classpath --glob) python example.py
其次讀取數(shù)據(jù)時(shí)，需要在數(shù)據(jù)的路徑前面加入HDFS前綴，比如：
hdfs://default/user/data/example.txt
三、分布式模型的示例代碼
該示例代碼是讀取HDFS上的MNIST數(shù)據(jù)，建立相應(yīng)的server與work集群構(gòu)建出一個(gè)三層的深度網(wǎng)絡(luò)，包含兩層卷積層以及一層SoftMax層。代碼如下：
from __future__ import 3/8 首頁 上一頁 1 2 3 4 5 6 下一頁 尾頁
