train_recpool_net.lua

----------------------------------------------------------------------
-- This script demonstrates how to define a training procedure,
-- irrespective of the model/loss functions chosen.
--
-- It shows how to:
--   + construct mini-batches on the fly
--   + define a closure to estimate (a noisy) loss
--     function, as well as its derivatives wrt the parameters of the
--     model to be trained
--   + optimize the function, according to several optmization
--     methods: SGD, L-BFGS.
--
-- Clement Farabet
----------------------------------------------------------------------

require 'torch'   -- torch
require 'xlua'    -- xlua provides useful tools, like progress bars
require 'optim'   -- an optimization package, for online and batch methods

local RecPoolTrainer = torch.class('nn.RecPoolTrainer')

local check_for_nans
local output_gradient_magnitudes

function RecPoolTrainer:__init(model, new_opt, layered_lambdas, track_criteria_outputs, receptive_field_builder)
   self.layered_lambdas = layered_lambdas
   self.track_criteria_outputs = track_criteria_outputs or false
   self.receptive_field_builder = receptive_field_builder -- if nil, then receptive fields are not built

   self:reset_options(new_opt)
   
   -- allowed output classes
   if new_opt.num_classification_categories == 10 then
      self.classes = {'1','2','3','4','5','6','7','8','9','0'}
   elseif new_opt.num_classification_categories == 2 then
      self.classes = {'right', 'left'}
   else
      error('unexpected number of classification categories: ' .. new_opt.num_classification_categories '; expected 10 or 2')
   end
   
   -- This matrix records the current confusion across classes
   self.confusion = optim.ConfusionMatrix(self.classes)
   self.current_performance = 0
   self.loss_hist = {}   
   self.grad_loss_hist = {}
   for i = 1,#model.criteria_list.criteria do
      self.loss_hist[i] = 0
      self.grad_loss_hist[i] = 0
   end
   self.num_zero_hist = {}
   
   -- Log results to files
   self.train_logger = optim.Logger(paths.concat(self.opt.log_directory, 'train.log'))
   --self.test_logger = optim.Logger(paths.concat(self.opt.log_directory, 'test.log'))

   -- Flatten the parameters (and gradParameters) into a single giant storage.  Each parameter and gradParameter tensor then views an offset into the common storage.  Shared parameters are only stored once, since Module:share() already sets the associated tensors to point to a common storage.
   if model then
      -- this must *NOT* be called twice; each call allocates new storage and unlinks the modules from the old storage(s)
      self.flattened_parameters,self.flattened_grad_parameters = model:getParameters()  
   else
      error('RecPoolTrainer requires a model')
   end
   self.model = model

   self.minibatch_inputs = torch.Tensor() -- these allow communication between the train function and the feval closure
   self.minibatch_targets = torch.Tensor()

   -- note that feval takes only current_params as input, whereas make_feval takes self as input; the self provided to make_feval is accessible to feval through the closure
   self.feval = self:make_feval() 
   self.epoch = 0
end

function RecPoolTrainer:reset_options(new_opt)
   -- set default options
   if not new_opt then
      new_opt = {}
   end
   self.opt = {}

   -- only copy the desired components from new_opt, and initialize any unspecified components to sensible defaults
   self.opt.log_directory = new_opt.log_directory or 'recpool_results' -- subdirectory in which to save/log experiments
   self.opt.visualize = new_opt.visualize or false -- visualize input data and weights during training
   self.opt.plot = new_opt.plot or false -- live plot
   self.opt.optimization = new_opt.optimization or 'SGD' -- optimization method: SGD | ASGD | CG | LBFGS
   self.opt.learning_rate = new_opt.learning_rate or 1e-3 -- learning rate at t=0
   self.opt.learning_rate_decay = new_opt.learning_rate_decay or 5e-7 -- should be adjusted based upon minibatch size in run_recpool_net
   self.opt.init_eval_counter = new_opt.init_eval_counter or 0
   self.opt.batch_size = new_opt.batch_size or 0 -- mini-batch size (0 = pure stochastic)
   self.opt.test_batch_size = new_opt.test_batch_size or 0 -- mini-batch size (0 = pure stochastic)
   self.opt.weight_decay = new_opt.weight_decay or 0 -- weight decay (SGD only)
   self.opt.L1_weight_decay = new_opt.L1_weight_decay or 0 -- weight decay (SGD only)
   self.opt.momentum = new_opt.momentum or 0 -- momentum (SGD only)
   self.opt.t0 = new_opt.t0 or 1 -- start averaging at t0 (ASGD only), where t0 is measured ASGD calls
   self.opt.max_iter = new_opt.max_iter or 2 -- maximum nb of iterations for CG and LBFGS
end

function RecPoolTrainer:reset_learning_rate(new_learning_rate)
   self.opt.learning_rate = new_learning_rate
   print('Resetting learning rate to ' .. new_learning_rate)
end

function RecPoolTrainer:get_flattened_parameters() -- flattened_parameters are more sensibly handled by the model, rather than the trainer
   return self.flattened_parameters
end

function RecPoolTrainer:get_output_flattened_parameters() -- flattened_parameters are more sensibly handled by the model, rather than the trainer
   if (self.opt.optimization == 'ASGD') and self.average_parameters then
      print('using averaged parameters')
      return self.average_parameters
   else
      return self.flattened_parameters
   end
end


-- create closure to evaluate f(X) and df/dX; the closure is necessary so minibatch_inputs and self.minibatch_targets are correct.  
-- self is provided by the column notation.  Thereafter, feval can be called without self as an argument, and the closure provides access to the (implicit) self argument of make_feval
function RecPoolTrainer:make_feval()
   --[[
   local internal_counter = 1
   --]]

   --[[ DEBUG CODE to check if minibatches produce the same gradients as pure SGD
   local batched_update = torch.Tensor()
   local batched_update_counter = 0
   --]]
      
   local feval = function(current_params)
      -- enforce all constraints on parameters, since the parameters are updated manually, rather than through updateParameters as generally expected
      if (self.opt.optimization ~= 'SGD') and (self.opt.optimization ~= 'ASGD') then -- if we only do one feval call per parameter update, then it is safe to repair once after the update
	 print('consider the need to repair on each iteration')
	 self.model:repair() -- THIS IS INEFFICIENT!  THIS DOUBLES THE TIME REQUIRED PER ITERATION!  THE COMPONENT OPERATIONS SHOULD BE IMPLEMENTED IN C!!!
      end

      -- get new parameters
      if current_params ~= self.flattened_parameters then 
	 self.flattened_parameters:copy(current_params)
	 print('copying parameters in feval') -- does this ever actually run?
      end
      
      -- Reset gradients.  This is more efficient than self.model:zeroGradParameters(), since gradParameters has all gradients flattened into a single storage, viewed by the many parameter tensors.  As a result, when parameters are shared by multiple modules, they are only zeroed once by this procedure.
      self.flattened_grad_parameters:zero() 
      
      -- total_err is the average of the error over the entire minibatch
      local total_err = 0
      
      -- evaluate function for complete minibatch
      --for i = 1,#self.minibatch_inputs do
      -- estimate total_err
      self.model:set_target(self.minibatch_targets)
      local err = self.model:updateOutput(self.minibatch_inputs)
      local output = self.model:get_classifier_output() -- while the model is a nn.Sequential, it terminates in a set of criteria
      total_err = total_err + err[1] -- the err returned by updateOutput is a tensor with one element, to maintain compatibility with ModfifiedJacobian
      
      --check_for_nans(self, output, 'outputs')
      
      -- estimate the gradient of the error with respect to the parameters: d total_err / dW
      self.model:updateGradInput(self.minibatch_inputs) -- gradOutput is not required, since all computation streams terminate in a criterion; implicitly pass nil
      self.model:accGradParameters(self.minibatch_inputs)
      
      -- update the confusion matrix.  This keeps track of the predicted output (maximum output conditional posterior probability) for each true output class
      if (self.minibatch_inputs:nDimension() == 1) and (type(self.minibatch_targets) == 'number') then -- minibatch contains a single element, so its components are of reduced dimensionality
	 self.confusion:add(output, self.minibatch_targets) 
      elseif (self.minibatch_inputs:nDimension() == 2) and (self.minibatch_targets:nDimension() == 1) then -- minibatch contains many elements, over which we must iterate
	 for j = 1,self.minibatch_inputs:size(1) do
	    self.confusion:add(output:select(1,j), self.minibatch_targets[j]) 
	 end
      else
	 error('dimensions of minibatch_inputs and minibatch_targets were not consistent')
      end
      
      -- track the evolution of sparsity and reconstruction errors
      if self.track_criteria_outputs then
	 for j = 1,#(self.model.criteria_list.criteria) do
	    if self.model.criteria_list.criteria[j].output then -- this need not be defined if we've disabled pooling or other layers
	       self.loss_hist[j] = self.model.criteria_list.criteria[j].output + self.loss_hist[j]
	       --print(self.model.criteria_list.names[j], self.model.criteria_list.criteria[j].gradInput)
	       if type(self.model.criteria_list.criteria[j].gradInput) == 'table' then
		  for k = 1,#self.model.criteria_list.criteria[j].gradInput do
		     self.grad_loss_hist[j] = self.model.criteria_list.criteria[j].gradInput[k]:norm() + self.grad_loss_hist[j]
		  end
	       else
		  self.grad_loss_hist[j] = self.model.criteria_list.criteria[j].gradInput:norm() + self.grad_loss_hist[j]
	       end
	    end -- if critera output exists
	 end -- for all criteria
      end --if track_criteria_outputs
      --end

      if self.receptive_field_builder then
	 self.receptive_field_builder:accumulate_shrink_weighted_inputs(self.minibatch_inputs, self.model.layers[1].module_list.shrink, self.model.layers[1].module_list.shrink_copies, 
									self.minibatch_targets)
      end

      -- normalize gradients and f(X)
      if self.minibatch_inputs:nDimension() == 2 then
	 -- we do not average gradients across a minibatch; rather we simply add them together.  This saves us from having to scale up the learning rate in proportion to the size of the minibatch
	 --self.flattened_grad_parameters:div(self.minibatch_inputs:size(1)) -- minibatches are stored along the rows; each row is a different minibatch element
	 total_err = total_err / self.minibatch_inputs:size(1)
      end

      --[[  DEBUG CODE to check if minibatches produce the same gradients as pure SGD
      if self.config.learningRate then
	 batched_update:resizeAs(self.flattened_grad_parameters)
	 if batched_update_counter == 0 then
	    batched_update:zero()
	 end
	 --batched_update:add(self.config.learningRate, self.flattened_grad_parameters)
	 batched_update:add(self.flattened_grad_parameters)
	 batched_update_counter = batched_update_counter + 1
	 if batched_update_counter == 10 then
	    batched_update_counter = 0
	    print('batched update magnitude is ' .. torch.norm(batched_update))
	 end
         --print('update magnitude is ' .. torch.norm(torch.mul(self.flattened_grad_parameters, self.config.learningRate)))
	 print('update magnitude is ' .. torch.norm(self.flattened_grad_parameters))
      end
      --]]

      --print('grad mag is ' .. self.flattened_grad_parameters:norm())
       
      --check_for_nans(self, self.flattened_grad_parameters, 'gradParameters')
      -- return f and df/dX
      return total_err, self.flattened_grad_parameters
   end
   
   return feval
end


function RecPoolTrainer:train(train_data, epoch_type) 
   -- epoch_type == 'validation' if we're just testing the network on the validation data, in which case nothing should be altered (in particular, learning should not be performed
   -- epoch_type == 'display' if we're generating figures, in which case we want to keep the order consistent
   if not(epoch_type) then
      self.epoch = self.epoch + 1
   end

   local this_epoch_batch_size = (((epoch_type == 'validation') or (epoch_type == 'display')) and self.opt.test_batch_size) or self.opt.batch_size
   print('using batch size ' .. this_epoch_batch_size .. ' from choices ' .. self.opt.test_batch_size .. ' and ' .. self.opt.batch_size)
   
   -- local vars
   local time = sys.clock()
   
   -- shuffle at each epoch
   local shuffle 
   if epoch_type == 'display' then
      shuffle = torch.range(1,train_data:nExample())
   else
      shuffle = torch.randperm(train_data:nExample()) --was trsize
   end

   -- do one epoch
   print('==> doing epoch on training data:')
   print("==> online epoch # " .. self.epoch .. ' [batch_size = ' .. this_epoch_batch_size .. ']')

   for t = 1, train_data:nExample(), math.max(1, this_epoch_batch_size) do
      -- disp progress
      if (t % 200 == 1) or (t == train_data:nExample()) then
	 xlua.progress(t, train_data:nExample())
      end
      
      -- create mini batch.  The minibatch_inputs and minibatch_targets elements of a RecPoolTrainer are viewed directly by the feval made by make_feval()
      -- previously, minibatches consisted of a table, the elements of which were the components of the minibatch.  However, this is not directly compatible with the torch facilities for efficiently processing minibatches.  Now, minibatches consist of tensors, with the elements along the rows.  An exception is made for minibatches of size one, which use tensors (or numbers) of reduced dimensionality, to take advantage of more efficient matrix-vector rather than matrix-matrix calculations.
      if this_epoch_batch_size == 0 then
	 self.minibatch_inputs = train_data.data[shuffle[t]]:double() -- This doesn't copy memory if the type is already correct
	 self.minibatch_targets = train_data.labels[shuffle[t]]
      else -- unlike with minibatches of size 1, we *ALWAYS* copy the data in forming larger minibatches.  This is necessary because tensors consist of regular strides within contiguous blocks of memory, rather than arbitrary arrays of pointers.
	 self.minibatch_inputs:resize(this_epoch_batch_size, train_data:dataSize())
	 if train_data:labelSize() == 1 then
	    self.minibatch_targets:resize(this_epoch_batch_size)
	 else
	    self.minibatch_targets:resize(this_epoch_batch_size, train_data:labelSize())
	 end
	 
	 -- load new samples for the batch
	 for i = 1,this_epoch_batch_size do 
	    --ensure that batches are full by wrapping around to the beginning if necessary; since the order of the dataset is reshuffled on each epoch, this shouldn't cause a big problem
	    local shuffle_index = ((t+i-2) % train_data:nExample()) + 1
	    --print('accessing element ' .. shuffle_index .. ', ' .. shuffle[shuffle_index])
	    self.minibatch_inputs:select(1,i):copy(train_data.data[shuffle[shuffle_index]]:double())
	    if train_data:labelSize() == 1 then
	       self.minibatch_targets[i] = train_data.labels[shuffle[shuffle_index]]
	    else
	       self.minibatch_targets:select(1,i):copy(train_data.labels[shuffle[shuffle_index]])
	    end
	 end
      end

      
      -- optimize on current mini-batch
      if ((epoch_type == 'validation') or (epoch_type == 'display')) then
	 self.model:prepare_test_batch()
	 --self.model:prepare_train_batch() -- DEBUG ONLY
	 self.feval(self.flattened_parameters) -- just run the network on the minibatch_inputs and minibatch_targets to generate the confusion matrix, without doing any training
	 -- this is necessary since some of these value are accessed by run_recpool_net.  Ideally, we should eliminate this break in the abstraction barrier
	 self.config = self.config or {evalCounter = self.opt.init_eval_counter or 0, 
				       learningRate = self.opt.learning_rate,
				       weightDecay = self.opt.weight_decay,
				       L1weightDecay = self.opt.L1_weight_decay,
				       momentum = self.opt.momentum,
				       learningRateDecay = self.opt.learning_rate_decay}
      else -- perform a train batch
	 self.model:prepare_train_batch()

	 if self.opt.optimization == 'CG' then
	    self.config = self.config or {maxIter = self.opt.max_iter}
	    optim.cg(self.feval, self.flattened_parameters, self.config)
	    
	 elseif self.opt.optimization == 'LBFGS' then
	    self.config = self.config or {learningRate = self.opt.learning_rate,
					  maxIter = self.opt.max_iter,
					  nCorrection = 10}
         optim.lbfgs(self.feval, self.flattened_parameters, self.config)
	 
	 elseif self.opt.optimization == 'SGD' then
	    if not(self.config) then
	       self.config = {evalCounter = self.opt.init_eval_counter or 0,
			      learningRate = self.opt.learning_rate,
			      weightDecay = self.opt.weight_decay,
			      L1weightDecay = self.opt.L1_weight_decay,
			      momentum = self.opt.momentum,
			      learningRateDecay = self.opt.learning_rate_decay} -- 5e-7
	       print('Initializing effective learning rate decay to ' .. self.config.evalCounter * self.config.learningRateDecay)
	    end
	    self.config.learningRate = self.opt.learning_rate -- make sure that the sgd learning rate reflects any resets
	    optim.sgd_decayed_weight_decay(self.feval, self.flattened_parameters, self.config)
	    
	 elseif self.opt.optimization == 'ASGD' then
	    self.config = self.config or {t = self.opt.init_eval_counter or 0,
					  eta0 = self.opt.learning_rate,
					  lambda = self.opt.learning_rate_decay / self.opt.learning_rate, -- matches the decay to that in SGD
					  t0 = self.opt.t0} -- measured in calls to ASGD
	    self.config.eta0 = self.opt.learning_rate -- make sure that asgd learning rate reflects any resets
	    self.config.lambda = self.opt.learning_rate_decay / self.opt.learning_rate
	    _,_,self.average_parameters = optim.asgd_no_weight_decay(self.feval, self.flattened_parameters, self.config)
	    
	 else
	    error('unknown optimization method')
	 end
	 -- repair the parameters one final time
	 self.model:repair() -- EFFICIENCY NOTE: Keep in mind that this is the most time consuming part of the operation!!!
      end -- perform a train batch
   end -- loop over the current epoch
   
   -- time taken for the current epoch (each call to train() only runs one epoch)
   time = sys.clock() - time
   time = time / train_data:nExample()
   print("==> time to learn 1 sample = " .. (time*1000) .. 'ms')
   
   print(self.confusion) -- print the confusion matrix for the current epoch
   self.current_performance = self.confusion.totalValid * 100 -- extract the current performance from the ConfusionMatrix object

   -- update logger/plot
   self.train_logger:add{['% mean class accuracy (train set)'] = self.confusion.totalValid * 100}
   if self.opt.plot then
      self.train_logger:style{['% mean class accuracy (train set)'] = '-'}
      self.train_logger:plot()
   end

   self.confusion:zero()

   for i = 1,#self.model.criteria_list.criteria do
      if self.track_criteria_outputs then
	 print('Criterion: ' .. self.model.criteria_list.names[i] .. ' = ' .. self.loss_hist[i]/train_data:nExample() .. '; grad = ' .. self.grad_loss_hist[i]/train_data:nExample())
	 self.loss_hist[i] = 0
	 self.grad_loss_hist[i] = 0
      end

      if (self.model.criteria_list.names[i] == 'pooling L2 shrink reconstruction loss') and not(self.model.disable_pooling) then
	 print('performing additional testing on ' .. self.model.criteria_list.names[i] .. ' with value ' .. self.model.criteria_list.criteria[i].output)
	 local alpha = self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda / self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda
	 local shrink_output = self.model.layers[1].module_list.shrink_copies[#self.model.layers[1].module_list.shrink_copies].output
	 local theoretical_shrink_reconstruction_loss = alpha^2 * math.pow(torch.norm(torch.cdiv(shrink_output, 
										torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha))), 2)
	 --print('dividing ', torch.mul(self.model.layers[1].module_list.shrink_copies[#self.model.layers[i].module_list.shrink_copies].output, alpha):unfold(1,10,10))
	 --print('by ', torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha):unfold(1,10,10))
	 local careful_reconstruction_loss = math.pow(torch.norm(torch.add(shrink_output, -1,
									   torch.cdiv(torch.cmul(shrink_output, torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2)), 
										      torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha)))), 2)
	 
	 local theoretical_shrink_position_loss = math.pow(torch.norm(torch.cdiv(torch.cmul(shrink_output, self.model.layers[1].module_list.decoding_pooling_dictionary.output), 
										 torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha))), 2)
	 
	 local combined_loss_careful = self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda * theoretical_shrink_reconstruction_loss + 
	    self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda * theoretical_shrink_position_loss
	 
	 -- this should not equal the exact combined loss, since (a + b)^2 ~= a^2 + b^2!!!
	 local combined_loss = self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda * 
	    math.pow(torch.norm(torch.cdiv(torch.cmul(shrink_output, 
						      torch.add(self.model.layers[1].module_list.decoding_pooling_dictionary.output, math.sqrt(alpha))), 
					      torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha))), 2)
	 local better_combined_loss = self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda * 
	    torch.sum(torch.cdiv(torch.pow(shrink_output, 2), 
				 torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha)))
	    -- THIS IS NO LONGER CORRECT, SINCE L1 SPARSITY AND L2 RECONSTRUCTION CRITERIA ARE REPEATED IN LISTA STACK
	 local exact_combined_loss = self.model.criteria_list.criteria[4].output + self.model.criteria_list.criteria[6].output -- this already includes the lambdas
	 print('*combined ratio is ' .. combined_loss / exact_combined_loss .. ' better: ' .. better_combined_loss / exact_combined_loss .. ' careful: ' .. combined_loss_careful / exact_combined_loss)
	    

	 print('shrink reconstruction ratio is ' .. (self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda * theoretical_shrink_reconstruction_loss) / self.model.criteria_list.criteria[i].output .. ' careful version ' .. (self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda * careful_reconstruction_loss) / self.model.criteria_list.criteria[i].output .. ' with criteria output ' .. self.model.criteria_list.criteria[i].output)
	 
	 print('shrink position ratio is ' .. (self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda * theoretical_shrink_position_loss) / self.model.criteria_list.criteria[6].output)


	 
	 -- version using alternative position loss ||sqrt(P*s)*x||^2
	 local theoretical_alt_shrink_reconstruction_loss = alpha^2 * math.pow(torch.norm(torch.cdiv(shrink_output, 
										torch.add(self.model.layers[1].module_list.decoding_pooling_dictionary.output, alpha))), 2)
	 local careful_alt_reconstruction_loss = math.pow(torch.norm(torch.add(shrink_output, -1,
								    torch.cdiv(torch.cmul(shrink_output, self.model.layers[1].module_list.decoding_pooling_dictionary.output), 
									       torch.add(self.model.layers[1].module_list.decoding_pooling_dictionary.output, alpha)))), 2)
	 local theoretical_alt_shrink_position_loss = math.pow(torch.norm(torch.cdiv(torch.cmul(shrink_output, torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 0.5)), 
										     torch.add(self.model.layers[1].module_list.decoding_pooling_dictionary.output, alpha))), 2)
	 local better_alt_combined_loss = self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda * 
	    torch.sum(torch.cdiv(torch.pow(shrink_output, 2), 
				 torch.add(self.model.layers[1].module_list.decoding_pooling_dictionary.output, alpha)))

	 print('*alt combined ratio is ' .. better_alt_combined_loss / exact_combined_loss)
	 print('alt shrink reconstruction ratio is ' .. (self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda * theoretical_alt_shrink_reconstruction_loss) / self.model.criteria_list.criteria[i].output .. ' careful version ' .. (self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda * careful_alt_reconstruction_loss) / self.model.criteria_list.criteria[i].output .. ' with criteria output ' .. self.model.criteria_list.criteria[i].output)
	 
	 print('alt shrink position ratio is ' .. (self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda * theoretical_alt_shrink_position_loss) / self.model.criteria_list.criteria[6].output)

	 

      end  -- pooling is enabled
   end	-- loop over all criteria 

   if not(self.model.disable_pooling) then
      local alpha = (self.layered_lambdas[1].pooling_L2_shrink_position_unit_lambda + self.layered_lambdas[1].pooling_L2_orig_position_unit_lambda) /
	 (self.layered_lambdas[1].pooling_L2_shrink_reconstruction_lambda + self.layered_lambdas[1].pooling_L2_orig_reconstruction_lambda)
      
      local theoretical_orig_position_loss = math.pow(torch.norm(torch.cdiv(torch.cmul(self.model.layers[1].module_list.decoding_feature_extraction_dictionary_transpose.output,
										       self.model.layers[1].module_list.decoding_pooling_dictionary.output), 
									    torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha))), 2)
      --[[
      local theoretical_orig_reconstruction_loss = math.pow(torch.norm(torch.add(INPUT, -1,
										 torch.cdiv(torch.cmul(shrink_output, torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2)),
											    torch.add(torch.pow(self.model.layers[1].module_list.decoding_pooling_dictionary.output, 2), alpha)))), 2)
      --]]

      print('orig position ratio is ' .. (self.layered_lambdas[1].pooling_L2_orig_position_unit_lambda * theoretical_orig_position_loss) / self.model.criteria_list.criteria[7].output)
      --print('orig reconstruction ratio is ' .. (self.layered_lambdas[1].pooling_L2_orig_reconstruction_lambda * theoretical_orig_reconstruction_loss) / self.model.criteria_list.criteria[5].output)
   end 
      
   print('Explaining away diag is', torch.diag(self.model.layers[1].module_list.explaining_away.weight):unfold(1,10,10))
   

   for i = 1,#self.model.layers do
      --print('feature reconstruction ', self.model.layers[i].module_list.decoding_feature_extraction_dictionary.output:unfold(1,10,10))
      --print('shrink magnitude ', self.model.layers[i].module_list.shrink.shrink_val:norm())
      --print('all shrink', self.model.layers[i].module_list.shrink.shrink_val:unfold(1,10,10))
      --print('explaining away diag', torch.diag(self.model.layers[i].module_list.explaining_away.weight):unfold(1,10,10))
      local single_shrink_output, single_offset_shrink_output, single_pooling_output
      if this_epoch_batch_size == 0 then
	 single_shrink_output = self.model.layers[i].module_list.shrink_copies[#self.model.layers[i].module_list.shrink_copies].output
	 if self.model.layers[i].module_list.offset_shrink then
	    single_offset_shrink_output = self.model.layers[i].module_list.offset_shrink.output
	 end
	 if not(self.model.disable_pooling) then 
	    single_pooling_output = self.model.layers[i].debug_module_list.pooling_seq.output[1]
	 end
      else -- only display the first element of the minibatch
	 single_shrink_output = self.model.layers[i].module_list.shrink_copies[#self.model.layers[i].module_list.shrink_copies].output:select(1,1)
	 if self.model.layers[i].module_list.offset_shrink then
	    single_offset_shrink_output = self.model.layers[i].module_list.offset_shrink.output:select(1,1)
	 end
	 if not(self.model.disable_pooling) then 
	    single_pooling_output = self.model.layers[i].debug_module_list.pooling_seq.output[1]:select(1,1)
	 end
      end

      print('final shrink output', single_shrink_output:unfold(1,10,10))
      if single_offset_shrink_output then 
	 print('offset shrink output', single_offset_shrink_output:unfold(1,10,10))
      end
      if not(self.model.disable_pooling) then -- this may not be defined if we've disabled pooling
	 print('pooling reconstruction', self.model.layers[i].module_list.decoding_pooling_dictionary.output:unfold(1,10,10))
      end
      -- these two outputs are from the middle of the processing chain, rather than the parameterized modules
      --print('pooling position units', self.model.layers[i].debug_module_list.compute_shrink_position_units.output:unfold(1,10,10))
      if not(self.model.disable_pooling) then -- this may not be defined if we've disabled pooling
	 print('pooling output', single_pooling_output:unfold(1,10,10))
      end
      -- since the sparsifying modules can be parameterized by lagrange multipliers, they are in the main module list
      if self.model.layers[i].module_list.feature_extraction_sparsifying_module.weight then
	 print('feature extraction L1', self.model.layers[i].module_list.feature_extraction_sparsifying_module.weight:unfold(1,10,10))
      end
      if self.model.layers[i].module_list.pooling_sparsifying_module.weight then
	 print('pooling L1', self.model.layers[i].module_list.pooling_sparsifying_module.weight:unfold(1,10,10))
      end
      if self.model.layers[i].module_list.mask_sparsifying_module.weight then
	 print('mask L1', self.model.layers[i].module_list.mask_sparsifying_module.weight:unfold(1,10,10))
      end
      --print('normalized output', self.model.layers[i].debug_module_list.normalize_output.output[1]:unfold(1,10,10))

      ---[[
      local m = self.model.layers[i].module_list.decoding_feature_extraction_dictionary.weight
      --local m = self.model.layers[i].module_list.decoding_pooling_dictionary.weight
      local norms = torch.Tensor(m:size(2))
      for j = 1,m:size(2) do
	 norms[j] = m:select(2,j):norm()
      end
      print('FE dec col norms are ', norms:unfold(1,10,10))

      --print('FE dec matrix is ', m)
      --]]

      --print('dec matrix is', self.model.layers[1].module_list.decoding_feature_extraction_dictionary.weight)
      --print('enc matrix is', self.model.layers[1].module_list.encoding_feature_extraction_dictionary.weight:t())
      --print('enc bias is', self.model.layers[1].module_list.encoding_feature_extraction_dictionary.bias:unfold(1,10,10))

      local m = self.model.layers[i].module_list.encoding_feature_extraction_dictionary.weight
      --local m = self.model.layers[i].module_list.encoding_pooling_dictionary.weight
      local norms = torch.Tensor(m:size(1))
      for j = 1,m:size(1) do
	 norms[j] = m:select(1,j):norm()
      end
      print('FE row norms are ', norms:unfold(1,10,10))

      --print('FE biases are ', self.model.layers[i].module_list.encoding_feature_extraction_dictionary.bias:unfold(1,10,10))

      --[[
      local m = self.model.layers[i].module_list.encoding_pooling_dictionary.weight
      local norms = torch.Tensor(m:size(1))
      for j = 1,m:size(1) do
	 norms[j] = torch.pow(m:select(1,j), 2):norm()
      end
      print('P row norms are ', norms:unfold(1,10,10))
      --]]
	 
      if self.model.layers[i].debug_module_list.normalize_pooled_output then
	 print('Pooling normalization is ', self.model.layers[i].debug_module_list.normalize_pooled_output.norm)
      end
     

      ---[[
      local m = self.model.layers[i].module_list.explaining_away.weight
      local norms = torch.Tensor(m:size(1))
      for j = 1,m:size(1) do
	 norms[j] = m:select(1,j):norm()
      end
      print('EXP row norms are ', norms:unfold(1,10,10))

      --print(m:select(1,101):unfold(1,10,10))
      --]]

      

      --[[
      print('shrink values', self.model.layers[i].module_list.shrink.shrink_val:unfold(1,10,10))
      --print('shrink values', torch.add(self.model.layers[i].module_list.shrink.shrink_val, -1e-5):unfold(1,10,10))
      --print('negative_shrink values', torch.add(self.model.layers[i].module_list.shrink.negative_shrink_val, 1e-5):unfold(1,10,10))
      -- display filters!  Also display reconstructions minus originals, so we can see how the reconstructions improve with training!
      -- check that without regularization, filters are meaningless.  Confirm that trainable pooling has an effect on the pooled filters.

      print('encoding_feature_extraction_dictionary output')
      print(self.model.layers[i].module_list.encoding_feature_extraction_dictionary.output:unfold(1,10,10))

      print('explaining_away outputs')
      local desired_size = 20 --self.model.layers[i].module_list.decoding_feature_extraction_dictionary.output:size(1)
      local explaining_away_output_tensor = torch.Tensor(desired_size, #self.model.layers[i].module_list.explaining_away_copies)
      for j = 1,#self.model.layers[i].module_list.explaining_away_copies do 	 
	 explaining_away_output_tensor:select(2,j):copy(self.model.layers[i].module_list.explaining_away_copies[j].output:narrow(1,1,desired_size))
      end
      print(explaining_away_output_tensor)

      print('raw shrink outputs')
      local desired_size = 20 --self.model.layers[i].module_list.decoding_feature_extraction_dictionary.output:size(1)
      local shrink_output_tensor = torch.Tensor(desired_size, #self.model.layers[i].module_list.shrink_copies)
      for j = 1,#self.model.layers[i].module_list.shrink_copies do 	 
	 shrink_output_tensor:select(2,j):copy(self.model.layers[i].module_list.shrink_copies[j].output:narrow(1,1,desired_size))
      end
      print(shrink_output_tensor)

      print('reconstructed shrink outputs')
      local desired_size = 200 --self.model.layers[i].module_list.decoding_feature_extraction_dictionary.output:size(1)
      local shrink_output_tensor = torch.Tensor(desired_size, #self.model.layers[i].module_list.shrink_copies)
      for j = 1,#self.model.layers[i].module_list.shrink_copies do 	 
	 shrink_output_tensor:select(2,j):copy(self.model.layers[i].module_list.decoding_feature_extraction_dictionary:updateOutput(
						  self.model.layers[i].module_list.shrink_copies[j].output):narrow(1,1,desired_size))
      end
      print(shrink_output_tensor)
      --]]
      
      local plot_recs = false
      if plot_recs then
	 if i == 1 then
	    plot_reconstructions(self.opt, self.minibatch_inputs, self.model.layers[i].module_list.decoding_feature_extraction_dictionary.output)
	 else
	    plot_reconstructions(self.opt, self.model.layers[i-1].debug_module_list.normalize_output.output[1], self.model.layers[i].module_list.decoding_feature_extraction_dictionary.output)
	 end
      end
   end -- loop over layers

   local m = self.model.module_list.classification_dictionary.weight
   local norms = torch.Tensor(m:size(1))
   local c_size = m:size(1)
   for j = 1,m:size(1) do
      norms[j] = m:select(1,j):norm()
   end
   print('C row norms are ', norms:unfold(1,c_size,c_size)) -- unfold so the tensor is presented horizontally rather than vertically
   
   if self.model.module_list.logsoftmax.output:nElement() > 0 then -- make sure the logsoftmax is used before trying to display its output
      if this_epoch_batch_size == 0 then
	 print('logsoftmax output is ', self.model.module_list.logsoftmax.output:unfold(1,c_size,c_size))
	 print('target is ' .. self.model.current_target)
      else
	 print('logsoftmax output is ', self.model.module_list.logsoftmax.output:select(1,1):unfold(1,c_size,c_size))
	 print('target is ', self.model.current_target[1])
      end
   end
   print('classification bias is ', self.model.module_list.classification_dictionary.bias:unfold(1,c_size,c_size))

   output_gradient_magnitudes(self)

   local index_list = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
   --local index_list = {11, 12, 13, 14, 15, 16, 4, 7, 8, 10, 42, 52, 63, 64, 67, 78}
   --local index_list = {1, 2, 3, 11, 12, 13, 4, 9, 21, 32, 72, 83, 88}
   --local index_list = {32, 34, 41, 58, 69, 70, 91, 103, 114, 121, 123, 138, 171, 201, 203, 213, 217, 238, 244, 290, 304, 327, 1, 2, 3, 4, 5, 6, 7, 8} -- for 11/30 400 units
   --local index_list = {30, 47, 53, 55, 77, 119, 172, 192, 196, 199, 206, 209, 232, 241, 255, 269, 290, 302, 303, 305, 311, 313, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} -- for 12/12 400 units manual
   --local index_list = {3, 7, 11, 16, 51, 77, 88, 107, 109, 127, 129, 1, 2, 4, 5, 8} -- 12/13 400 units 11 ista iterations 5e-3 training rate
   local num_shrink_output_tensor_elements = #index_list -- self.model.layers[1].module_list.shrink.output:size(1)
   local shrink_output_tensor = torch.Tensor(num_shrink_output_tensor_elements, 1 + #self.model.layers[1].module_list.shrink_copies)

   for j = 1,#index_list do
      -- automatically choose whether to use 1D or 2D indexing into shrink.output, depending upon the size of the minibatch
      shrink_output_tensor[{j, 1}] = self.model.layers[1].module_list.shrink.output[((this_epoch_batch_size == 0) and {index_list[j]}) or {1,index_list[j]}]
   end
   
   for i = 1,#self.model.layers[1].module_list.shrink_copies do
      --shrink_output_tensor:select(2,i):copy(self.model.layers[1].module_list.shrink_copies[i].output)
      for j = 1,#index_list do
	 -- automatically choose whether to use 1D or 2D indexing into shrink_copies[i].output, depending upon the size of the minibatch
	 shrink_output_tensor[{j, i+1}] = self.model.layers[1].module_list.shrink_copies[i].output[((this_epoch_batch_size == 0) and {index_list[j]}) or {1,index_list[j]}]
      end
   end
   print('evolution of selected shrink elements', shrink_output_tensor)

   local exp_away_grad_mags = torch.Tensor(#self.model.layers[1].module_list.explaining_away_copies)
   for j = 1,#self.model.layers[1].module_list.explaining_away_copies do 	 
      exp_away_grad_mags[j] = self.model.layers[1].module_list.explaining_away_copies[j].gradInput:norm()
   end
   print('explaining away grad mags are ', exp_away_grad_mags)

   --[[
   local class_crit_index = 0
   for j = 1,#(self.model.criteria_list.criteria) do
      if self.model.criteria_list.names[j] == 'classification criterion' then
	 class_crit_index = j
	 print('classification criterion grad is', self.model.criteria_list.criteria[j].gradInput)
      end
   end
   print('pre-classification grad is ', self.model.module_list.logsoftmax.gradInput)
   local grad_inp = self.model.criteria_list.criteria[class_crit_index].gradInput
   local grad_inp_sum = grad_inp:sum(2):select(2,1)
   local exp_val = torch.exp(self.model.module_list.classification_dictionary.output)
   print('correct pre-class grad is ', torch.add(grad_inp, torch.epandAs(-1 * grad_inp_sum, torch.div(exp_val, exp_val:sum(2):select(2,1)), grad_inp))) 
   --]]

   print('pre-classification grad mag is ' .. self.model.module_list.logsoftmax.gradInput:norm())
   print('post-classification grad mag is ' .. self.model.module_list.classification_dictionary.gradInput:norm())

   print('pre-classification grad mag is ', self.model.module_list.logsoftmax.gradInput)
   print('logsoftmax output is ', self.model.module_list.logsoftmax.output)


   for j = 1,#(self.model.criteria_list.criteria) do
      if self.model.criteria_list.criteria[j].output then -- this need not be defined if we've disabled pooling or other layers
	 local criteria_norm = 0
	 if type(self.model.criteria_list.criteria[j].gradInput) == 'table' then
	    for k = 1,#self.model.criteria_list.criteria[j].gradInput do
	       criteria_norm = criteria_norm + self.model.criteria_list.criteria[j].gradInput[k]:norm()
	    end
	 else
	    criteria_norm = self.model.criteria_list.criteria[j].gradInput:norm()
	 end
	 print(self.model.criteria_list.names[j], criteria_norm)
      end -- if critera output exists
   end -- for all criteria


   --[[
   -- save/log current net
   local filename = paths.concat(self.opt.log_directory, 'model.net')
   os.execute('mkdir -p ' .. sys.dirname(filename))
   print('==> saving model to '..filename)
   torch.save(filename, self.model)
   --]]

end


function output_gradient_magnitudes(self)
   for i = 1,#self.model.layers do
      if self.model.layers[i].module_list.shrink.grad_shrink_val then  -- THIS IS A TOTAL HACK!!!
	 print('layer ' .. i, 'encoding FE dict', self.model.layers[i].module_list.encoding_feature_extraction_dictionary.gradWeight:norm(), 
	       'decoding FE dict', self.model.layers[i].module_list.decoding_feature_extraction_dictionary.gradWeight:norm(),
	       'shrink', self.model.layers[i].module_list.shrink.grad_shrink_val:norm(), 'explaining away', self.model.layers[i].module_list.explaining_away.gradWeight:norm(),
	       'encoding P dict', self.model.layers[i].module_list.encoding_pooling_dictionary.gradWeight:norm(), 
	       'decoding P dict', self.model.layers[i].module_list.decoding_pooling_dictionary.gradWeight:norm())
      else
	 local agg_norm = 1
	 print('layer ' .. i, 'encoding FE dict', self.model.layers[i].module_list.encoding_feature_extraction_dictionary.gradWeight:norm(agg_norm) /
	       self.model.layers[i].module_list.encoding_feature_extraction_dictionary.gradWeight:nElement(), 
	       'decoding FE dict', self.model.layers[i].module_list.decoding_feature_extraction_dictionary.gradWeight:norm(agg_norm) / 
		  self.model.layers[i].module_list.decoding_feature_extraction_dictionary.gradWeight:nElement(),
	       'explaining away', self.model.layers[i].module_list.explaining_away.gradWeight:norm(agg_norm) / 
		  self.model.layers[i].module_list.explaining_away.gradWeight:nElement())
	 if not(self.model.disable_pooling) then
	    print('encoding P dict', self.model.layers[i].module_list.encoding_pooling_dictionary.gradWeight:norm(agg_norm) / 
		  self.model.layers[i].module_list.encoding_pooling_dictionary.gradWeight:nElement(), 
		  'decoding P dict', self.model.layers[i].module_list.decoding_pooling_dictionary.gradWeight:norm(agg_norm) / 
		     self.model.layers[i].module_list.decoding_pooling_dictionary.gradWeight:nElement())
	 end
      end
   end
   print('classification layer', 'class dict', self.model.module_list.classification_dictionary.gradWeight:norm())

   --print(self.model.layers[1].module_list.encoding_pooling_dictionary.gradWeight:unfold(1,10,10))
   --print(self.model.layers[1].debug_module_list.ista_sparsifying_loss_seq.output[1]:unfold(1,10,10))
   --print(self.model.layers[1].debug_module_list.pooling_L2_loss_seq.gradInput[1]:unfold(1,10,10))
   --print('max value is: ', torch.pow(self.model.layers[1].debug_module_list.ista_sparsifying_loss_seq.output[1], 2):max(), self.model.layers[1].debug_module_list.pooling_L2_loss_seq.gradInput[1]:max())
   
end

function check_for_nans(self, output, name)
   local found_a_nan = false
   
   local function find_nans(x)
      if x ~= x then
	 found_a_nan = true
      end
   end

   local function find_nans_in_table(x)
      for k,v in pairs(x) do
	 found_a_nan = false
	 v:apply(find_nans)
	 if found_a_nan then
	    print('found a nan in entry ' .. k)
	    print(v:unfold(1,10,10))
	 end
      end
   end

   --[[
      if internal_counter % 100 == 1 then
      print(output:unfold(1,10,10))
      print(self.model.encoding_feature_extraction_dictionary.weight[{1,{1,10}}]:unfold(1,10,10))
      print(self.model.explaining_away.weight[{1,{1,10}}]:unfold(1,10,10))
      print(self.model.shrink.shrink_val[{{1,10}}]:unfold(1,10,10))
      print(self.model.encoding_pooling_dictionary.weight[{1,{1,10}}]:unfold(1,10,10))
      print(self.model.classification_dictionary.weight[{1,{1,10}}]:unfold(1,10,10))
      end
      internal_counter = internal_counter + 1
   --]]
      
   output:apply(find_nans)
   if found_a_nan then
      for i = 1,#self.model.layers do
	 print('checking for nans in ' .. name .. ' layer ' .. i)
	 io.read()
	 print('outputs')
	 --print(output:unfold(1,10,10))
	 print(self.model.layers[i].module_list.encoding_feature_extraction_dictionary.output:unfold(1,10,10))
	 print(self.model.layers[i].module_list.encoding_pooling_dictionary.output:unfold(1,10,10))
	 print(self.model.layers[i].debug_module_list.ista_sparsifying_loss_seq.output[1]:unfold(1,10,10))
	 print(self.model.layers[i].debug_module_list.pooling_seq.output[1]:unfold(1,10,10)) -- one nan is present
	 print(self.model.layers[i].debug_module_list.pooling_L2_loss_seq.output[1]:unfold(1,10,10))
	 print(self.model.layers[i].debug_module_list.pooling_sparsifying_loss_seq.output[1]:unfold(1,10,10))
	 print(self.model.layers[i].module_list.decoding_pooling_dictionary.output:unfold(1,10,10))
      end
      print(self.model.module_list.classification_dictionary.output:unfold(1,10,10))
      io.read()

      for i = 1,#self.model.layers do
	 print('gradInputs test in layer ' .. i)

	 print('shrink_reconstruction')
	 find_nans_in_table(self.model.layers[i].debug_module_list.compute_shrink_reconstruction_loss_seq.gradInput)
	 print('orig_reconstruction')
	 find_nans_in_table(self.model.layers[i].debug_module_list.compute_orig_reconstruction_loss_seq.gradInput)
	 print('shrink position_loss')
	 find_nans_in_table(self.model.layers[i].debug_module_list.compute_shrink_position_loss_seq.gradInput)
	 print('orig position_loss')
	 find_nans_in_table(self.model.layers[i].debug_module_list.compute_orig_position_loss_seq.gradInput)
	 io.read()
	 
	 print('gradInputs second test')
	 print('shrink_rec_numerator')
	 find_nans_in_table(self.model.layers[i].debug_module_list.construct_shrink_rec_numerator_seq.gradInput)
	 --print(self.model.construct_shrink_rec_numerator_seq.output:unfold(1,10,10))
	 print('shink_pos_numerator_seq')
	 find_nans_in_table(self.model.layers[i].debug_module_list.construct_shrink_pos_numerator_seq.gradInput)
	 --print(self.model.construct_shrink_pos_numerator_seq.output:unfold(1,10,10))
	 print('orig_rec_numerator_seq')
	 find_nans_in_table(self.model.layers[i].debug_module_list.construct_orig_rec_numerator_seq.gradInput)
	 --print(self.model.construct_orig_rec_numerator_seq.output:unfold(1,10,10))
	 print('orig_pos_numerator_seq')
	 find_nans_in_table(self.model.layers[i].debug_module_list.construct_orig_pos_numerator_seq.gradInput)
	 --print(self.model.construct_orig_pos_numerator_seq.output:unfold(1,10,10))
	 print('denominator_seq')
	 find_nans_in_table(self.model.layers[i].debug_module_list.construct_denominator_seq.gradInput)
	 --print(self.model.construct_denominator_seq.output:unfold(1,10,10))
	 io.read()


	 --[[
	 print('gradInputs')
	 print(self.model.encoding_feature_extraction_dictionary.gradInput:unfold(1,10,10)) -- all nans
	 print(self.model.encoding_pooling_dictionary.gradInput:unfold(1,10,10)) -- all nans
	 print(self.model.ista_sparsifying_loss_seq.gradInput[1]:unfold(1,10,10)) -- all nans
	 print(self.model.pooling_seq.gradInput[1]:unfold(1,10,10)) -- all nans
	 print('pooling L2 loss input 1')
	 print(self.model.pooling_L2_loss_seq.gradInput[1]:unfold(1,10,10)) -- all nans
	 print('pooling L2 loss input 2')
	 print(self.model.pooling_L2_loss_seq.gradInput[2]:unfold(1,10,10)) -- all nans
	 print('pooling L2 loss input 3')
	 print(self.model.pooling_L2_loss_seq.gradInput[3]:unfold(1,10,10)) -- all nans
	 print(self.model.pooling_sparsifying_loss_seq.gradInput[1]:unfold(1,10,10))
	 print(self.model.pooling_sparsifying_loss_seq.gradInput[2]:unfold(1,10,10))
	 print(self.model.classification_dictionary.gradInput:unfold(1,10,10))
	 io.read()
	 
	 print('weights')
	 print(self.model.encoding_feature_extraction_dictionary.weight[{1,{1,10}}]:unfold(1,10,10))
	 print(self.model.explaining_away.weight[{1,{1,10}}]:unfold(1,10,10))
	 print(self.model.shrink.shrink_val[{{1,10}}]:unfold(1,10,10))
	 print(self.model.encoding_pooling_dictionary.weight[{1,{1,10}}]:unfold(1,10,10))
	 print(self.model.classification_dictionary.weight)
	 io.read()
	 --]]
      end -- loop over model layers
   end -- if found_a_nan
end