Improved detail of epoch timing reports, and performance measurement now uses only the model timings

Fixed performance issue with Tensor::pokeLastDimension
Added a benchmark for studying the weak scaling of a fully-connected deep neural net, with cmdline tunable sizes
Added a stopwatch/timer class for easier timing

Author: giltirn

benchmark/Makefile.am

@@ -3,7 +3,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/include
 AM_LDFLAGS = -Wl,-rpath=$(prefix)/lib
 LDADD = -L$(top_builddir)/src -lhpcortex
 benchmarkdir = $(prefix)/benchmark
-benchmark_PROGRAMS = benchmark_matrixBatchTensorContractRight benchmark_dnn benchmark_matrixTensorContractComponent benchmark_activated_deriv benchmarkBatchTensorDNNcomponentStep benchmark_tensor_reorder benchmark_thin_mul_mat_mattranspose benchmark_axpy_mat_thinmat benchmark_matrixBatchTensorContractLeft benchmark_batch3tensorContract benchmark_mul_mattranspose_thinmat benchmark_batchTensorContractToMatrix benchmark_matrixBatchTensorAxpy benchmark_decoder benchmark_multihead_self_attention 
+benchmark_PROGRAMS = benchmark_matrixBatchTensorContractRight benchmark_dnn benchmark_matrixTensorContractComponent benchmark_activated_deriv benchmarkBatchTensorDNNcomponentStep benchmark_tensor_reorder benchmark_thin_mul_mat_mattranspose benchmark_axpy_mat_thinmat benchmark_dnn_weak_scaling benchmark_matrixBatchTensorContractLeft benchmark_batch3tensorContract benchmark_mul_mattranspose_thinmat benchmark_batchTensorContractToMatrix benchmark_matrixBatchTensorAxpy benchmark_decoder benchmark_multihead_self_attention 
 benchmark_matrixBatchTensorContractRight_SOURCES = benchmark_matrixBatchTensorContractRight.cpp 
 benchmark_matrixBatchTensorContractRight_LDADD = $(LDADD)
 benchmark_matrixBatchTensorContractRight_LINK = $(CXXLD) $(AM_LDFLAGS) $(LDFLAGS) -o $@
@@ -28,6 +28,9 @@ benchmark_thin_mul_mat_mattranspose_LINK = $(CXXLD) $(AM_LDFLAGS) $(LDFLAGS) -o
 benchmark_axpy_mat_thinmat_SOURCES = benchmark_axpy_mat_thinmat.cpp 
 benchmark_axpy_mat_thinmat_LDADD = $(LDADD)
 benchmark_axpy_mat_thinmat_LINK = $(CXXLD) $(AM_LDFLAGS) $(LDFLAGS) -o $@
+benchmark_dnn_weak_scaling_SOURCES = benchmark_dnn_weak_scaling.cpp 
+benchmark_dnn_weak_scaling_LDADD = $(LDADD)
+benchmark_dnn_weak_scaling_LINK = $(CXXLD) $(AM_LDFLAGS) $(LDFLAGS) -o $@
 benchmark_matrixBatchTensorContractLeft_SOURCES = benchmark_matrixBatchTensorContractLeft.cpp 
 benchmark_matrixBatchTensorContractLeft_LDADD = $(LDADD)
 benchmark_matrixBatchTensorContractLeft_LINK = $(CXXLD) $(AM_LDFLAGS) $(LDFLAGS) -o $@

benchmark/benchmark_dnn_weak_scaling.cpp

@@ -0,0 +1,78 @@
+//A strong-scaling benchmark using a deep fully-connected network of variable size
+//The total batch size is scaled to the number of ranks
+
+#include<HPCortex.hpp>
+
+
+int main(int argc, char** argv){
+  initialize(argc,argv);
+  communicators().reportSetup();
+    
+  int hidden_layers = 5;
+  int input_features = 32;
+  int output_features = 32;
+  int hidden_neurons = 128;
+  int rank_batch_size = 32;
+  int nbatch_per_epoch = 100;
+  int nepoch = 10;
+
+  int arg=1;
+  while(arg < argc){
+    std::string sarg(argv[arg]);
+    if(sarg == "--hidden_layers"){
+      hidden_layers = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else if(sarg == "--input_features"){
+      input_features = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else if(sarg == "--output_features"){
+      output_features = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else if(sarg == "--hidden_neurons"){
+      hidden_neurons = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else if(sarg == "--rank_batch_size"){
+      rank_batch_size = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else if(sarg == "--nbatch_per_epoch"){
+      nbatch_per_epoch = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else if(sarg == "--nepoch"){
+      nepoch = std::stoi(argv[arg+1]);
+      arg+=2;
+    }else{
+      std::cout << "Unknown cmdline argument: " << sarg << std::endl;
+      assert(0);
+    }
+  }
+  
+  
+  std::mt19937 rng(1234);
+
+  auto model_body = enwrap( dnn_layer(hidden_neurons, input_features, ReLU<float>(),
+				 input_layer< confSingle, Matrix<float> >()
+				 )
+		       );
+  for(int l=0;l<hidden_layers-2;l++) //the first dnn_layer has one hidden layer, as does the last
+    model_body = enwrap( dnn_layer(hidden_neurons,hidden_neurons, ReLU<float>(), std::move(model_body)) );
+
+  auto model = dnn_layer(output_features, hidden_neurons, noActivation<float>(), model_body);
+
+  std::cout << "Params: " << model.nparams() << std::endl;
+  
+  int nrank = communicators().ddpNrank();
+  int ndata = rank_batch_size * nrank;
+  std::vector<XYpair<float,1,1> > data(ndata);
+  for(auto &d : data){
+    d.x = Vector<float>(input_features);
+    d.y = Vector<float>(output_features);
+    uniformRandom(d.x,rng);
+    uniformRandom(d.y,rng);
+  }
+
+  auto loss = mse_cost(model);
+  
+  AdamOptimizer<float> opt(0.01);
+  XYpairDataLoader<float,1,1> loader(data);
+  train(loss, loader, opt, nepoch, ndata);
+}

include/Timing.hpp

@@ -21,5 +21,45 @@ inline double since(const std::chrono::system_clock::time_point &when){
   return double(usCountSince(when)) / 1e6;
 }
 
+/**
+ * @brief A simple timer class
+ */
+class Timer{
+private:
+  double ttot;
+  std::chrono::system_clock::time_point tp;
+public:
+  /**
+   * @brief Reset the accumulated time to zero and (optionally) begin timing from now
+   */
+  inline void restart(bool start = false){
+    ttot = 0;
+    if(start) tp = now();
+  }
+  /**
+   * @brief Resume a timer when paused
+   */
+  inline void resume(){
+    tp = now();
+  }
+  /**
+   * @brief Pause a timer, adding the time since start/resume to the accumulated time
+   */
+  inline void pause(){
+    ttot += since(tp);
+  }
+  
+  /**
+   * @brief Get the accumulated time 
+   */
+  inline double time() const{
+    return ttot;
+  }
 
+  /**
+   * @brief Construct and (optionally) start the timer
+   */  
+  Timer(bool start_on_create = false){ restart(start_on_create); }
+};
 
+#define TIME(into, ...) into.resume(); __VA_ARGS__; into.pause(); 

include/implementation/Optimizers.tcc

@@ -1,3 +1,26 @@
+struct train_stage_time_report{
+  Timer total;
+  Timer loader;
+  Timer model;
+  Timer reduce;
+
+  train_stage_time_report(): total(true){}
+
+  std::string report(double FLOPS){
+    double Gflops = FLOPS/1e9 /model.time();
+    std::ostringstream os;
+    double other = total.time() - loader.time() - model.time() - reduce.time();
+    os << "loader: " << loader.time() << "s, "
+       << "model: " << model.time() << "s (" << Gflops << " Gflops), "
+       << "reduction: " << reduce.time() << "s, "
+       << "other: " << other << "s | "
+       << "total: " << total.time();
+    return os.str();
+  }
+};
+
+
+
 template<typename DataLoader, typename LossWrappedModelType, typename Optimizer>
 std::pair<
 std::vector<typename LossWrappedModelType::FloatType>,
@@ -50,8 +73,8 @@ train(LossWrappedModelType &loss_func, const DataLoader &train_data, DataLoader
     //////////// train epoch ///////////////
     optimizer.epochStart(epoch, do_print);
     std::random_shuffle ( didx_train.begin(), didx_train.end(), [&](const int l){ return dist(gen); }  ); //shuffle training data indices
-    FloatType lmax_train=std::numeric_limits<FloatType>::lowest(),  lmin_train = std::numeric_limits<FloatType>::max(),  lavg_train = 0.;
-    auto ts=now();
+    FloatType lmax_train=std::numeric_limits<FloatType>::lowest(),  lmin_train = std::numeric_limits<FloatType>::max(),  lavg_train = 0.;    
+    train_stage_time_report t_train;
 
     for(int block=0;block<nblocks_ddp_train;block++){
       int ddp_blocksize_actual = std::min(nbatch_train - block*ddp_blocksize, ddp_blocksize);
@@ -63,14 +86,21 @@ train(LossWrappedModelType &loss_func, const DataLoader &train_data, DataLoader
 	int bidx = block*ddp_blocksize + me; //which batch are we doing?
 
 	//Get the batch
-	auto bxy = train_data.batch(didx_train.data() + bidx*batch_size, batch_size);
+	TIME(t_train.loader,
+	     auto bxy = train_data.batch(didx_train.data() + bidx*batch_size, batch_size);
+	     );
 
+	TIME(t_train.model,
 	loss = loss_func.loss(bxy.x, bxy.y, DerivYes);
 	deriv = loss_func.deriv();
+	     );
       }
+
+      TIME(t_train.reduce, 
       ddpAverage(&loss,1,false); //no need to bcast the loss to the pipeline ranks
       ddpAverage(deriv,true); //share the deriv over all pipeline ranks
-           
+	   )
+	   
       //if(do_print) std::cout << epoch << "-" << block << " : "<< loss << std::endl;
       lmax_train = std::max(lmax_train,loss);
       lmin_train = std::min(lmin_train,loss);
@@ -84,45 +114,59 @@ train(LossWrappedModelType &loss_func, const DataLoader &train_data, DataLoader
       losses_train[block+nblocks_ddp_train*epoch] = loss;
     }
     lavg_train /= nblocks_ddp_train;
-    double train_time = since(ts);
-    double train_Tflops = nbatch_train * double(loss_func.FLOPS(0) + loss_func.FLOPS(1) + 2*loss_func.nparams())/1.0e12 / train_time;
+
+    t_train.total.pause();
+    double train_FLOPS = nbatch_train * double(loss_func.FLOPS(0) + loss_func.FLOPS(1));
 
     //////////// end train epoch ///////////////
 
     //////////// validate epoch ///////////////
     if(valid_data){
       FloatType lmax_valid=std::numeric_limits<FloatType>::lowest(),  lmin_valid = std::numeric_limits<FloatType>::max(),  lavg_valid = 0.;
-      ts=now();
+      train_stage_time_report t_valid;
 
       for(int block=0;block<nblocks_ddp_valid;block++){
 	int ddp_blocksize_actual = std::min(nbatch_valid - block*ddp_blocksize, ddp_blocksize);
 
 	FloatType loss = 0;
 	if(me < ddp_blocksize_actual){ 
-	  int bidx = block*ddp_blocksize + me; 
+	  int bidx = block*ddp_blocksize + me;
+	  TIME(t_valid.loader,
 	  auto bxy = valid_data->batch(didx_valid.data() + bidx*batch_size, batch_size); //no need to shuffle
+	       );
+	  TIME(t_valid.model,
 	  loss = loss_func.loss(bxy.x, bxy.y, DerivNo);
+	       );
 	}
-      
+
+	TIME(t_valid.reduce,
 	ddpAverage(&loss,1,false);
-          
+	     );
+	     
 	lmax_valid = std::max(lmax_valid,loss);
 	lmin_valid = std::min(lmin_valid,loss);
 	lavg_valid += loss;
 
 	losses_valid[block+nblocks_ddp_valid*epoch] = loss;
-      }
+      }      
       lavg_valid /= nblocks_ddp_valid;
-      double valid_time = since(ts);
-      double valid_Tflops = nbatch_valid * double(loss_func.FLOPS(0))/1.0e12 / valid_time;
+      t_valid.total.pause();
+      
+      double valid_FLOPS = nbatch_valid * double(loss_func.FLOPS(0));
 
       //////////// end validate epoch ///////////////
 
       if(do_print) std::cout << "Epoch : " << epoch << std::endl
-			     << "training time : " << train_time <<"s (" << train_Tflops << " Tflops) loss min: " << lmin_train << " avg: " << lavg_train << " max: " << lmax_train << std::endl
-			     << "validation time : " << valid_time <<"s (" << valid_Tflops << " Tflops) loss min: " << lmin_valid << " avg: " << lavg_valid << " max: " << lmax_valid << std::endl;
-    }else{ //if not validating, just print info on the training losses    
-      if(do_print) std::cout << "Epoch : " << epoch << " time : " << train_time <<"s ("<< train_Tflops << " Tflops) loss min: " << lmin_train << " avg: " << lavg_train << " max: " << lmax_train << std::endl;    
+			     << "training loss min: " << lmin_train << " avg: " << lavg_train << " max: " << lmax_train << std::endl
+			     << "validation loss min: " << lmin_valid << " avg: " << lavg_valid << " max: " << lmax_valid << std::endl
+			     << "training timings: " << t_train.report(train_FLOPS) << std::endl
+			     << "validation timings: " << t_valid.report(valid_FLOPS) << std::endl;
+
+      
+    }else{ //if not validating, just print info on the training losses
+      if(do_print) std::cout << "Epoch : " << epoch << std::endl
+			     << "loss min: " << lmin_train << " avg: " << lavg_train << " max: " << lmax_train << std::endl
+			     << "timings: " << t_train.report(train_FLOPS) << std::endl;
     }
   }//epoch
 

include/implementation/Tensors.tcc

@@ -72,7 +72,7 @@ void Tensor<FloatType,Dim>::pokeLastDimension(const Tensor<FloatType,Dim-1> &ins
 
   autoView(ins_v,ins,DeviceRead);
   autoView(t_v,(*this),DeviceReadWrite);
-  accelerator_for2d(dummy1,1, i,other_size,32,{
+  accelerator_for_gen(1,0,splitBlock<32>(), i,other_size,{
       t_v.data()[idx + size_last *i] = ins_v.data()[i];
     });
 }