Foreground / Mask / Background

local net
local netD
local mask_net
local motion_net 
local static_net
local penalty_net 
if opt.finetune == '' then -- build network from scratch
  net = nn.Sequential()

  static_net = nn.Sequential()
  static_net:add(nn.View(-1, 100, 1, 1))
  static_net:add(nn.SpatialFullConvolution(100, 512, 4,4))
  static_net:add(nn.SpatialBatchNormalization(512)):add(nn.ReLU(true))
  static_net:add(nn.SpatialFullConvolution(512, 256, 4,4, 2,2, 1,1))
  static_net:add(nn.SpatialBatchNormalization(256)):add(nn.ReLU(true))
  static_net:add(nn.SpatialFullConvolution(256, 128, 4,4, 2,2, 1,1))
  static_net:add(nn.SpatialBatchNormalization(128)):add(nn.ReLU(true))
  static_net:add(nn.SpatialFullConvolution(128, 64, 4,4, 2,2, 1,1))
  static_net:add(nn.SpatialBatchNormalization(64)):add(nn.ReLU(true))
  static_net:add(nn.SpatialFullConvolution(64, 3, 4,4, 2,2, 1,1))
  static_net:add(nn.Tanh())

  local net_video = nn.Sequential()
  net_video:add(nn.View(-1, 100, 1, 1, 1))
  net_video:add(nn.VolumetricFullConvolution(100, 512, 2,4,4))
  net_video:add(nn.VolumetricBatchNormalization(512)):add(nn.ReLU(true))
  net_video:add(nn.VolumetricFullConvolution(512, 256, 4,4,4, 2,2,2, 1,1,1))
  net_video:add(nn.VolumetricBatchNormalization(256)):add(nn.ReLU(true))
  net_video:add(nn.VolumetricFullConvolution(256, 128, 4,4,4, 2,2,2, 1,1,1))
  net_video:add(nn.VolumetricBatchNormalization(128)):add(nn.ReLU(true))
  net_video:add(nn.VolumetricFullConvolution(128, 64, 4,4,4, 2,2,2, 1,1,1))
  net_video:add(nn.VolumetricBatchNormalization(64)):add(nn.ReLU(true))

  local mask_out = nn.VolumetricFullConvolution(64,1, 4,4,4, 2,2,2, 1,1,1)
  penalty_net = nn.L1Penalty(opt.lambda, true)
  mask_net = nn.Sequential():add(mask_out):add(nn.Sigmoid()):add(penalty_net) 
  gen_net = nn.Sequential():add(nn.VolumetricFullConvolution(64,3, 4,4,4, 2,2,2, 1,1,1)):add(nn.Tanh())
  net_video:add(nn.ConcatTable():add(gen_net):add(mask_net))

  -- [1] is generated video, [2] is mask, and [3] is static
  net:add(nn.ConcatTable():add(net_video):add(static_net)):add(nn.FlattenTable())

Video的size为[batch_size, time, channel, height, width]，图中表示为：height*width*time(channel)。

Convolution参数为(input_channel, kernel_number, [kernel_size], [strides], [paddings])，SpatialFullConvolution中kernel_size等为2d，VolumetricFullConvolution中kernel_size等为3d。

View(): 相当于reshape

ConcatTable():

                  +-----------+
             +----> {member1, |
+-------+    |    |           |
| input +----+---->  member2, |
+-------+    |    |           |
   or        +---->  member3} |
 {input}          +-----------+

Generated Video

  -- video .* mask (with repmat on mask)
  motion_net = nn.Sequential():add(nn.ConcatTable():add(nn.SelectTable(1))
                                                   :add(nn.Sequential():add(nn.SelectTable(2))
                                                                       :add(nn.Squeeze())
                                                                       :add(nn.Replicate(3, 2)))) -- for color chan 
                              :add(nn.CMulTable())

  -- static .* (1-mask) (then repmatted)
  local sta_part = nn.Sequential():add(nn.ConcatTable():add(nn.Sequential():add(nn.SelectTable(3))
                                                                           :add(nn.Replicate(opt.frameSize, 3))) -- for time
                                                       :add(nn.Sequential():add(nn.SelectTable(2))
                                                                           :add(nn.Squeeze())
                                                                           :add(nn.MulConstant(-1))
                                                                           :add(nn.AddConstant(1))
                                                                           :add(nn.Replicate(3, 2)))) -- for color chan
                                  :add(nn.CMulTable())

  net:add(nn.ConcatTable():add(motion_net):add(sta_part)):add(nn.CAddTable())

SelectTable(i): 选择ConcatTable中的第i个member。
Replicate(n, dim): 在第dim个维度复制为n个
(a, b, c):nn.Replicate(3, 2)-->(a, 3, b, c)

Discriminator

  netD = nn.Sequential()

  netD:add(nn.VolumetricConvolution(3,64, 4,4,4, 2,2,2, 1,1,1))
  netD:add(nn.LeakyReLU(0.2, true))
  netD:add(nn.VolumetricConvolution(64,128, 4,4,4, 2,2,2, 1,1,1))
  netD:add(nn.VolumetricBatchNormalization(128,1e-3)):add(nn.LeakyReLU(0.2, true))
  netD:add(nn.VolumetricConvolution(128,256, 4,4,4, 2,2,2, 1,1,1))
  netD:add(nn.VolumetricBatchNormalization(256,1e-3)):add(nn.LeakyReLU(0.2, true))
  netD:add(nn.VolumetricConvolution(256,512, 4,4,4, 2,2,2, 1,1,1))
  netD:add(nn.VolumetricBatchNormalization(512,1e-3)):add(nn.LeakyReLU(0.2, true))
  netD:add(nn.VolumetricConvolution(512,2, 2,4,4, 1,1,1, 0,0,0))
  netD:add(nn.View(2):setNumInputDims(4)) 

Discriminator Optimizer

-- optimization closure
-- the optimizer will call this function to get the gradients
local data_im,data_label
local fDx = function(x)
  gradParametersD:zero()

  -- fetch data
  data_tm:reset(); data_tm:resume()
  data_im = data:getBatch()
  data_tm:stop()

  -- ship to GPU
  noise:normal()
  target:copy(data_im)
  label:fill(real_label) 
  -- real_label=1, fake_label=2

  -- forward/backwards real examples
  local output = netD:forward(target)
  errD = criterion:forward(output, label)
  local df_do = criterion:backward(output, label)
  netD:backward(target, df_do)

  -- generate fake examples
  local fake = net:forward(noise)
  target:copy(fake)
  label:fill(fake_label)

  -- forward/backwards fake examples
  local output = netD:forward(target)
  errD = errD + criterion:forward(output, label)
  local df_do = criterion:backward(output, label)
  netD:backward(target, df_do)

  errD = errD / 2

  return errD, gradParametersD
end

Generator Optimizer

local fx = function(x)
  gradParameters:zero()

  label:fill(real_label)
  local output = netD.output
  err = criterion:forward(output, label)
  local df_do = criterion:backward(output, label)
  local df_dg = netD:updateGradInput(target, df_do)

  net:backward(noise, df_dg)

  return err, gradParameters
end

使用updateGradInput是为了不对netD进行backpropagation，仅仅是用链式法则计算梯度的中间过程。