Caffe::Snapshot的运行过程

Snapshot的存储

概述

Snapshot的存储格式有两种，分别是BINARYPROTO格式和hdf5格式。BINARYPROTO是一种二进制文件，并且可以通过修改shapshot_format来设置存储类型。该项的默认是BINARYPROTO。不管哪种格式，运行的过程是类似的，都是从Solver<Dtype>::Snapshot()函数进入，首先调用Net网络的方法，再操作网络中的每一层，最后再操作每一层中blob，最后调用write函数写入输出。源码入口：

 void Solver<Dtype>::Snapshot() {
   CHECK(Caffe::root_solver());
   string model_filename;
   switch (param_.snapshot_format()) {
   case caffe::SolverParameter_SnapshotFormat_BINARYPROTO:
     model_filename = SnapshotToBinaryProto();
     break;
   case caffe::SolverParameter_SnapshotFormat_HDF5:
     model_filename = SnapshotToHDF5();
     break;
   default:
     LOG(FATAL) << "Unsupported snapshot format.";
   }

BINARYPROTO格式

如果是BINARYPROTO的存储格式，就执行如下代码：

 string Solver<Dtype>::SnapshotToBinaryProto() {
   string model_filename = SnapshotFilename(".caffemodel");
   LOG(INFO) << "Snapshotting to binary proto file " << model_filename;
   NetParameter net_param;
   net_->ToProto(&net_param, param_.snapshot_diff());
   WriteProtoToBinaryFile(net_param, model_filename);
   return model_filename;
 }   

首先会执行SnapshotFilename(“.caffemodel”)函数，识别出sovler.prototxt文件中snapshot_prefix的内容，作用该snapshot文件的文件名前缀。然后调用net_->ToProto()，具体的代码如下：

 void Net<Dtype>::ToProto(NetParameter* param, bool write_diff) const {
   param->Clear();
   param->set_name(name_);
   for (int i = ; i < net_input_blob_indices_.size(); ++i) {
     param->add_input(blob_names_[net_input_blob_indices_[i]]);
   }
   for (int i = ; i < layers_.size(); ++i) {
     LayerParameter* layer_param = param->add_layer();
     layers_[i]->ToProto(layer_param, write_diff);
   }
 }  

获取到网络中的每层的名字等参数后，调用layers_[i]->ToProto()每一层的ToProto方法，接下来

 void Layer<Dtype>::ToProto(LayerParameter* param, bool write_diff) {
   param->Clear();
   param->CopyFrom(layer_param_);
   param->clear_blobs();
   for (int i = ; i < blobs_.size(); ++i) {
     blobs_[i]->ToProto(param->add_blobs(), write_diff);
   }
 } 

然后调用当前层下的所有blob的ToProto方法，即：

 void Blob<double>::ToProto(BlobProto* proto, bool write_diff) const {
   proto->clear_shape();
   for (int i = ; i < shape_.size(); ++i) {
     proto->mutable_shape()->add_dim(shape_[i]);
   }
   proto->clear_double_data();
   proto->clear_double_diff();
   const double* data_vec = cpu_data();
   for (int i = ; i < count_; ++i) {
     proto->add_double_data(data_vec[i]);
   }
   if (write_diff) {
     const double* diff_vec = cpu_diff();
     for (int i = ; i < count_; ++i) {
       proto->add_double_diff(diff_vec[i]);
     }
   }

在每一个blob中，会调用add_double_data()函数，把data添加到snapshot文件中，同时会判断是否当前blob参与diff的计算，如果需要当前blob需要diff参数，就调用add_double_diff()添加到snapshot文件中。

调用完所有的blob的ToProto()方法后，会执行WriteProtoToBinaryFile()把该文件写出即可。

 void WriteProtoToBinaryFile(const Message& proto, const char* filename) {
   fstream output(filename, ios::out | ios::trunc | ios::binary);
   CHECK(proto.SerializeToOstream(&output));
 }

在该方法里调用FStream的output方法进行输出。

Hdf5格式

Hdf5格式的运行过程和BINARYPROTO格式的过程类似，首先会调用SnapshotToHDF5()函数，即：

 string Solver<Dtype>::SnapshotToHDF5() {
   string model_filename = SnapshotFilename(".caffemodel.h5");
   LOG(INFO) << "Snapshotting to HDF5 file " << model_filename;
   net_->ToHDF5(model_filename, param_.snapshot_diff());
   return model_filename;
 }

首先会执行SnapshotFilename(“.caffemodel.h5”)函数，识别出sovler.prototxt文件中snapshot_prefix的内容，作用该snapshot文件的文件名前缀。然后调用net_->ToHDF5()，即：

 void Net<Dtype>::ToHDF5(const string& filename, bool write_diff) const {
   hid_t file_hid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT,
       H5P_DEFAULT);
   hid_t data_hid = H5Gcreate2(file_hid, "data", H5P_DEFAULT, H5P_DEFAULT,
       H5P_DEFAULT);
     hid_t diff_hid = -;
   if (write_diff) {
     diff_hid = H5Gcreate2(file_hid, "diff", H5P_DEFAULT, H5P_DEFAULT,
         H5P_DEFAULT);
    }
   for (int layer_id = ; layer_id < layers_.size(); ++layer_id) {
     const LayerParameter& layer_param = layers_[layer_id]->layer_param();
     string layer_name = layer_param.name();
     hid_t layer_data_hid = H5Gcreate2(data_hid, layer_name.c_str(),
     hid_t layer_diff_hid = -;
     if (write_diff) {
       layer_diff_hid = H5Gcreate2(diff_hid, layer_name.c_str(),
           H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
  }
     int num_params = layers_[layer_id]->blobs().size();
     for (int param_id = ; param_id < num_params; ++param_id) {
       ostringstream dataset_name;
       dataset_name << param_id;
       const int net_param_id = param_id_vecs_[layer_id][param_id];
       if (param_owners_[net_param_id] == -) {
         hdf5_save_nd_dataset<Dtype>(layer_data_hid, dataset_name.str(),
             *params_[net_param_id]);
       }
       if (write_diff) {
         hdf5_save_nd_dataset<Dtype>(layer_diff_hid, dataset_name.str(),
             *params_[net_param_id], true);
       }
 ...............
 H5Fclose(file_hid);
 }

该函数首先调用H5Fcreate()创建一个file文件，然后循环调用每一层，通过调用每一层的H5Gcreate2函数记录出该层的data_hid或者diff_hid(如果该层需要参与计算)，然后进入每一层内部的blob，然后在当前blob内调用hdf5_save_nd_dataset()或hdf5_save_nd_dataset()(如果当前blob需要参与计算diff)，将data添加到hdf5格式的文件中，最后调用H5Fclose(file_hid)函数，输出该文件。

Snapshot的恢复

概述

想在已经训练好的网络上继续训练，那么需要调用Restore()方法从snapshot的文件中恢复成网络，从而缩短了训练时间。方法的入口是Solver<Dtype>::Restore(const char* state_file)函数，即：

 void Solver<Dtype>::Restore(const char* state_file) {
   CHECK(Caffe::root_solver());
   string state_filename(state_file);
   if (state_filename.size() >=  &&
       state_filename.compare(state_filename.size() - , , ".h5") == ) {
     RestoreSolverStateFromHDF5(state_filename);
   } else {
     RestoreSolverStateFromBinaryProto(state_filename);
   }

该函数会解析snapshot文件是BINARYPROTO格式还是Hdf5格式，如果是BINARYPROTO格式的话就调用RestoreSolverStateFromBinaryProto()函数，如果格式Hdf5的格式，就执行RestoreSolverStateFromHDF5()。

BINARYPROOTO格式

如果是BINARYPROTO格式，则执行下列代码：

 void SGDSolver<Dtype>::RestoreSolverStateFromBinaryProto(
     const string& state_file) {
   SolverState state;
   ReadProtoFromBinaryFile(state_file, &state);
   this->iter_ = state.iter();
   if (state.has_learned_net()) {
     NetParameter net_param;
     ReadNetParamsFromBinaryFileOrDie(state.learned_net().c_str(), &net_param);
     this->net_->CopyTrainedLayersFrom(net_param);
   }
   this->current_step_ = state.current_step();
   CHECK_EQ(state.history_size(), history_.size())
       << "Incorrect length of history blobs.";
   for (int i = ; i < history_.size(); ++i) {
     history_[i]->FromProto(state.history(i));
   }
 }

该函数会大量调用google的protobuf包内的函数，首先会通过ReadProtoFromBinaryFile()函数读取BINARYPROTO格式的文件来返回是否可以成功读取。然后判断该snapshot是否有曾经训练过的网络，如果有，则调用函数ReadNetParamsFromBinaryFileOrDie()读取出该Net网络，然后调用函数CopyTrainedLayersFrom(net_param)具体恢复该网络的每一层以及当前层内的所有blob，具体数据恢复的工作就是CopyTrainedLayersFrom()函数内部变量调用FromProto()函数来实现blob复制的。然后会通过函数current_step()来判断上次训练的位置(迭代到多少次)，然后通过循环把训练过的data数据通过FromProto()完成数据的复制。

Hdf5格式

 void SGDSolver<Dtype>::RestoreSolverStateFromHDF5(const string& state_file) {
   hid_t file_hid = H5Fopen(state_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
   CHECK_GE(file_hid, ) << "Couldn't open solver state file " << state_file;
   this->iter_ = hdf5_load_int(file_hid, "iter");
   if (H5LTfind_dataset(file_hid, "learned_net")) {
     string learned_net = hdf5_load_string(file_hid, "learned_net");
     this->net_->CopyTrainedLayersFrom(learned_net);
   }
   this->current_step_ = hdf5_load_int(file_hid, "current_step");
   hid_t history_hid = H5Gopen2(file_hid, "history", H5P_DEFAULT);
   CHECK_GE(history_hid, ) << "Error reading history from " << state_file;
   int state_history_size = hdf5_get_num_links(history_hid);
   CHECK_EQ(state_history_size, history_.size())
       << "Incorrect length of history blobs.";
   for (int i = ; i < history_.size(); ++i) {
     ostringstream oss;
     oss << i;
     hdf5_load_nd_dataset<Dtype>(history_hid, oss.str().c_str(), ,
                                 kMaxBlobAxes, history_[i].get());
   }
   H5Gclose(history_hid);
   H5Fclose(file_hid);
 }

该函数会识别hdf5格式存储的snapshot文件的file_hid编号，会判断是否存在之前训练过的网络，如果存在则执行CopyTrainedLayersFrom()函数，完成网络的每层以及每层内的blob的数据的恢复复制，然后或取上一次的训练位置(进行的迭代)，并且调用函数hdf5_load_nd_dataset()具体把每次迭代的数据恢复复制，最后再调用H5Fclose()关闭。