一言蔽之，这个层就是用来算loss的，作者定义的multi task loss
loss 形式.png
对外主要提供6个函数：
- layer make_region_layer(int batch, int h, int w, int n, int classes, int coords);
- void forward_region_layer(const layer l, network net);
- void backward_region_layer(const layer l, network net);
- void get_region_boxes(layer l, int w, int h, float thresh, float **probs, box *boxes, int only_objectness, int *map, float tree_thresh, int nomult);
- void resize_region_layer(layer *l, int w, int h);
- void zero_objectness(layer l);
layer make_region_layer(...)##
这个函数主要用来初始化 region_layer,参数详情见 layer
layer make_region_layer(int batch, int w, int h, int n, int classes, int coords)
{
layer l = {0};//初始化层
l.type = REGION;//说明这是 REGION 层
l.n = n;//一个 cell 提出 n 个box
l.batch = batch;
l.h = h;
l.w = w;
l.c = n*(classes + coords + 1);// 有 n * (classes + 4 + 1) 个channel
l.out_w = l.w;
l.out_h = l.h;
l.out_c = l.c;
l.classes = classes;//有多少物体类别
l.coords = coords;//4 x,y,w,h
l.cost = calloc(1, sizeof(float));//cost , only 1
l.biases = calloc(n*2, sizeof(float));//box 的长宽先验值 ，故有 2*number of boxes
l.bias_updates = calloc(n*2, sizeof(float));//box 长宽先验值的 update
l.outputs = h*w*n*(classes + coords + 1);//输出 feature map 大小,e.g. whole boxes 13*13*5*(20+4+1)
l.inputs = l.outputs;//
l.truths = 30*(5);//大概只有30 个ground truth，每个5个boxes？
l.delta = calloc(batch*l.outputs, sizeof(float));//batch*outputs
l.output = calloc(batch*l.outputs, sizeof(float));//batch*outputs
int i;
// 初始化为0.5 长宽先验值
for(i = 0; i < n*2; ++i){
l.biases[i] = .5;
}
l.forward = forward_region_layer;//函数指针指定所使用的函数
l.backward = backward_region_layer;//函数指针指定所使用的函数
#ifdef GPU
l.forward_gpu = forward_region_layer_gpu;
l.backward_gpu = backward_region_layer_gpu;
l.output_gpu = cuda_make_array(l.output, batch*l.outputs);
l.delta_gpu = cuda_make_array(l.delta, batch*l.outputs);
#endif
fprintf(stderr, "detection\n");
srand(0);
return l;
}
这里需要说明一下这个biases,它是预先设定的长宽先验值，从聚类中得到。
若不提供先验值，则全部初始化为0.5。
void resize_region_layer(...)##
将 feature map resize，调整其宽长以及宽长所影响的其它参数，目测只能往大里resize，不然数据丢失
void resize_region_layer(layer *l, int w, int h)
{
l->w = w;//新宽度
l->h = h;//新高度
//更改w，h影响到的其它变量值改变
l->outputs = h*w*l->n*(l->classes + l->coords + 1);
l->inputs = l->outputs;
//realloc
l->output = realloc(l->output, l->batch*l->outputs*sizeof(float));
l->delta = realloc(l->delta, l->batch*l->outputs*sizeof(float));
#ifdef GPU
cuda_free(l->delta_gpu);
cuda_free(l->output_gpu);
l->delta_gpu = cuda_make_array(l->delta, l->batch*l->outputs);
l->output_gpu = cuda_make_array(l->output, l->batch*l->outputs);
#endif
}
void forward_region_layer(...)##
这个函数就是前馈计算损失函数的，下面分部分介绍这个重点函数：
1.把前一层卷积结果拿来
void forward_region_layer(const layer l, network net)
{
int i,j,b,t,n;
//参见network.c，每次执行前馈时，都把 net.input 指向当前层的输出
memcpy(l.output, net.input, l.outputs*l.batch*sizeof(float));//把最后的卷积结果拷贝过来
本层用到的数据.png
本层用到的数据长这样，但是注意它是保存到一维数组中的，所以这里又涉及到一个索引定位问题，见下面函数,给出第几个batch，这个batch中数据的位置（一般指向第k个box的初始位置），和要索引的是哪个特征元素，即可得其坐标位置。本质还是4维到1维的映射。
| entry | 意义 |
|---|---|
| 0 | x |
| 1 | y |
| 2 | w |
| 3 | h |
| 4 | confidence |
| 5 | class |
int entry_index(layer l, int batch, int location, int entry)
{
int n = location / (l.w*l.h);
int loc = location % (l.w*l.h);
return batch*l.outputs + n*l.w*l.h*(l.coords+l.classes+1) + entry*l.w*l.h + loc;
}
index 定位示意图
2.把所有的x，y，confidence使用逻辑斯蒂函数映射到（0,1）
#ifndef GPU
for (b = 0; b < l.batch; ++b){
for(n = 0; n < l.n; ++n){//how many boxes!!!
int index = entry_index(l, b, n*l.w*l.h, 0);//locate bth batch nth box's x,y channel
activate_array(l.output + index, 2*l.w*l.h, LOGISTIC);//x,y coord convert to (0,1) by implement logistic
index = entry_index(l, b, n*l.w*l.h, 4);//locate bth batch nth box's confidence channel
activate_array(l.output + index, l.w*l.h, LOGISTIC);//confidence convert to (0,1) by implement logistic
}
}
下面是计算class概率的两种方式，根据设置选其中一个：
3.1.采用 wordtree 的形式，映射每个class的概率
这其实类似Trie树，整体概率->条件概率，沿着树乘下去
if (l.softmax_tree){//temporary ignore ,tree structure classification
int i;
int count = 5;
for (i = 0; i < l.softmax_tree->groups; ++i) {
int group_size = l.softmax_tree->group_size[i];
softmax_cpu(net.input + count, group_size, l.batch, l.inputs, l.n*l.w*l.h, 1, l.n*l.w*l.h, l.temperature, l.output + count);
count += group_size;
}
}
word tree 示意图
3.2.普通的 softmax 映射分类结果
对每个batch的每个cell的每个anchor box的 class predict 进行 softmax映射
else if (l.softmax){//now we use
int index = entry_index(l, 0, 0, 5);//apply softmax to n classes scores
//according to softmax_cpu function(in blas.c),calculate softmax to each batches' each boxes' position's softmax
//drawing pictures helps understand
softmax_cpu(net.input + index, l.classes, l.batch*l.n, l.inputs/l.n, l.w*l.h, 1, l.w*l.h, 1, l.output + index);
}
#endif
下面就是针对每个元素计算和理想（ground truth）之间的差值了
4.1.初始化各种值
memset(l.delta, 0, l.outputs * l.batch * sizeof(float));//allocate memory and init gradient*lambda which is delta
if(!net.train) return;
float avg_iou = 0;//平均 IOU
float recall = 0; //平均召回率
float avg_cat = 0;
float avg_obj = 0;//有物体的 predict平均
float avg_anyobj = 0;//所有predict 平均 indicate all boxes having objects' probability
int count = 0;
int class_count = 0;
*(l.cost) = 0;
**4.2.计算softmax_tree形式下的 loss **
for (b = 0; b < l.batch; ++b) {
//now do not use softmax tree
if(l.softmax_tree){
int onlyclass = 0;
for(t = 0; t < 30; ++t){
box truth = float_to_box(net.truth + t*5 + b*l.truths, 1);
if(!truth.x) break;
int class = net.truth[t*5 + b*l.truths + 4];
float maxp = 0;
int maxi = 0;
if(truth.x > 100000 && truth.y > 100000){
for(n = 0; n < l.n*l.w*l.h; ++n){
int class_index = entry_index(l, b, n, 5);
int obj_index = entry_index(l, b, n, 4);
float scale = l.output[obj_index];
l.delta[obj_index] = l.noobject_scale * (0 - l.output[obj_index]);
float p = scale*get_hierarchy_probability(l.output + class_index, l.softmax_tree, class, l.w*l.h);
if(p > maxp){
maxp = p;
maxi = n;
}
}
int class_index = entry_index(l, b, maxi, 5);
int obj_index = entry_index(l, b, maxi, 4);
delta_region_class(l.output, l.delta, class_index, class, l.classes, l.softmax_tree, l.class_scale, l.w*l.h, &avg_cat);
if(l.output[obj_index] < .3) l.delta[obj_index] = l.object_scale * (.3 - l.output[obj_index]);
else l.delta[obj_index] = 0;
++class_count;
onlyclass = 1;
break;
}
}
if(onlyclass) continue;
}
4.3.1.计算noobj时的confidence loss
对每个box，找其所对应的 ground truth box，原则是 IOU 大于阈值则匹配为ground truth，此时其loss为0，否则为（0 - l.output[obj_index]）
时间复杂度O（hwn*GTboxes）
noobj loss part
//calculate 2 parts loss:confidence loss & region box loss
for (j = 0; j < l.h; ++j) {
for (i = 0; i < l.w; ++i) {
for (n = 0; n < l.n; ++n) {
int box_index = entry_index(l, b, n*l.w*l.h + j*l.w + i, 0);
//generally it create pred boxes,whose coords is percent of whole images
box pred = get_region_box(l.output, l.biases, n, box_index, i, j, l.w, l.h, l.w*l.h);
float best_iou = 0;
//for above pred box,find the truth_box whose iou is the biggest
//one pred box <-> one ground truth box
for(t = 0; t < 30; ++t){
box truth = float_to_box(net.truth + t*5 + b*l.truths, 1);
if(!truth.x) break;
float iou = box_iou(pred, truth);
if (iou > best_iou) {
best_iou = iou;
}
}
//probability of whether a box is an object
int obj_index = entry_index(l, b, n*l.w*l.h + j*l.w + i, 4);
avg_anyobj += l.output[obj_index];
//calculate loss
//if best_iou > threshold,which means this boxes matches one ground truth ,this is an object no loss
//else loss= -noobject_scale*probability ,because we hope this -> 0
l.delta[obj_index] = l.noobject_scale * (0 - l.output[obj_index]);
if (best_iou > l.thresh) {
l.delta[obj_index] = 0;
}
另外这里面用到了一个获取region box的函数，可以结合下图来一起理解
box get_region_box(float *x, float *biases, int n, int index, int i, int j, int w, int h, int stride)
{
box b;
b.x = (i + x[index + 0*stride]) / w;//占整体的比例，此处w，h为最后一层feature的宽高
b.y = (j + x[index + 1*stride]) / h;
b.w = exp(x[index + 2*stride]) * biases[2*n] / w;
b.h = exp(x[index + 3*stride]) * biases[2*n+1] / h;
return b;
}
get region box
4.3.2.计算region loss
//what does net.seen means for?
//记录网络看了多少张图片了，希望其具有稳定性
//net.seen += net.batch
if(*(net.seen) < 12800){
box truth = {0};
truth.x = (i + .5)/l.w;//(i,j)th box center
truth.y = (j + .5)/l.h;//(i,j)th box center
truth.w = l.biases[2*n]/l.w;//percent of w
truth.h = l.biases[2*n+1]/l.h;//percent of w
delta_region_box(truth, l.output, l.biases, n, box_index, i, j, l.w, l.h, l.delta, .01, l.w*l.h);
}
}
}
}
这里面用到了delta_region_box函数
float delta_region_box(box truth, float *x, float *biases, int n, int index, int i, int j, int w, int h, float *delta, float scale, int stride)
{
box pred = get_region_box(x, biases, n, index, i, j, w, h, stride);
float iou = box_iou(pred, truth);
float tx = (truth.x*w - i);
float ty = (truth.y*h - j);
float tw = log(truth.w*w / biases[2*n]);
float th = log(truth.h*h / biases[2*n + 1]);
delta[index + 0*stride] = scale * (tx - x[index + 0*stride]);
delta[index + 1*stride] = scale * (ty - x[index + 1*stride]);
delta[index + 2*stride] = scale * (tw - x[index + 2*stride]);
delta[index + 3*stride] = scale * (th - x[index + 3*stride]);
return iou;
}
region delta
注意结合上面的，如果网络看了少于12800张图片，则:
| parameters | val |
|---|---|
| tx | 0.5 |
| ty | 0.5 |
| tw | 0 |
| th | 0 |
大概是为了稳定性
4.3.3.计算region box loss
第一眼看去，这居然和上面的 4.2.2 一毛一样，然而仔细看看还是有区别的：
其关键点在于，我们挑选的pred box 不一样！！！
首先遍历 ground truth box，然后从 ground truth box 中心点所在的那个 cell 的 n 个 pred boxes 中找到 IOU 最大的 box 来计算 loss！
region box loss
for(t = 0; t < 30; ++t){
box truth = float_to_box(net.truth + t*5 + b*l.truths, 1);
if(!truth.x) break;
float best_iou = 0;
int best_n = 0;
i = (truth.x * l.w);
j = (truth.y * l.h);
//printf("%d %f %d %f\n", i, truth.x*l.w, j, truth.y*l.h);
box truth_shift = truth;
truth_shift.x = 0;
truth_shift.y = 0;//ground truth 平移到（0,0）
//printf("index %d %d\n",i, j);
for(n = 0; n < l.n; ++n){
int box_index = entry_index(l, b, n*l.w*l.h + j*l.w + i, 0);
box pred = get_region_box(l.output, l.biases, n, box_index, i, j, l.w, l.h, l.w*l.h);
if(l.bias_match){
pred.w = l.biases[2*n]/l.w;
pred.h = l.biases[2*n+1]/l.h;
}
//printf("pred: (%f, %f) %f x %f\n", pred.x, pred.y, pred.w, pred.h);
pred.x = 0;
pred.y = 0;//pred 平移到（0,0）
float iou = box_iou(pred, truth_shift);
if (iou > best_iou){
best_iou = iou;
best_n = n;
}
}
//printf("%d %f (%f, %f) %f x %f\n", best_n, best_iou, truth.x, truth.y, truth.w, truth.h);
//max iou for loss calcuation
int box_index = entry_index(l, b, best_n*l.w*l.h + j*l.w + i, 0);
float iou = delta_region_box(truth, l.output, l.biases, best_n, box_index, i, j, l.w, l.h, l.delta, l.coord_scale * (2 - truth.w*truth.h), l.w*l.h);
if(iou > .5) recall += 1;
avg_iou += iou;
注意这时匹配IOU，是把 ground truth box 和 pred box 都平移到中心坐标为（0,0）来计算 ，也即去除中心偏移带来的影响
4.3.4.计算有 object 框框的 confidence loss
has object confidence loss
//l.delta[best_index + 4] = iou - l.output[best_index + 4];
int obj_index = entry_index(l, b, best_n*l.w*l.h + j*l.w + i, 4);
avg_obj += l.output[obj_index];
//loss for has object_scale;
l.delta[obj_index] = l.object_scale * (1 - l.output[obj_index]);
if (l.rescore) {
l.delta[obj_index] = l.object_scale * (iou - l.output[obj_index]);
}
根据预先设置是否rescore，看回归到1还是iou
4.3.5.类别回归的 loss
class loss
int class = net.truth[t*5 + b*l.truths + 4];
if (l.map) class = l.map[class];
int class_index = entry_index(l, b, best_n*l.w*l.h + j*l.w + i, 5);
//class loss
delta_region_class(l.output, l.delta, class_index, class, l.classes, l.softmax_tree, l.class_scale, l.w*l.h, &avg_cat);
++count;
++class_count;
}
}
//printf("\n");
//calculate whole loss
*(l.cost) = pow(mag_array(l.delta, l.outputs * l.batch), 2);
printf("Region Avg IOU: %f, Class: %f, Obj: %f, No Obj: %f, Avg Recall: %f, count: %d\n", avg_iou/count, avg_cat/class_count, avg_obj/count, avg_anyobj/(l.w*l.h*l.n*l.batch), recall/count, count);
}
这里面用到了 delta_region_class 函数
void delta_region_class(float *output, float *delta, int index, int class, int classes, tree *hier, float scale, int stride, float *avg_cat)
{
int i, n;
if(hier){// word tree类型的loss
float pred = 1;
while(class >= 0){
pred *= output[index + stride*class];
int g = hier->group[class];
int offset = hier->group_offset[g];
for(i = 0; i < hier->group_size[g]; ++i){
delta[index + stride*(offset + i)] = scale * (0 - output[index + stride*(offset + i)]);
}
delta[index + stride*class] = scale * (1 - output[index + stride*class]);
class = hier->parent[class];
}
*avg_cat += pred;
} else {//这就是普通的loss
for(n = 0; n < classes; ++n){
delta[index + stride*n] = scale * (((n == class)?1 : 0) - output[index + stride*n]);
if(n == class) *avg_cat += output[index + stride*n];
}
}
}
至此完成所有delta的计算任务！！！
void backward_region_layer(...)##
在 forward 中已经计算好 delta 了，所以 backward 不做操作
void backward_region_layer(const layer l, network net)
{
/*
int b;
int size = l.coords + l.classes + 1;
for (b = 0; b < l.batch*l.n; ++b){
int index = (b*size + 4)*l.w*l.h;
gradient_array(l.output + index, l.w*l.h, LOGISTIC, l.delta + index);
}
axpy_cpu(l.batch*l.inputs, 1, l.delta, 1, net.delta, 1);
*/
}
void get_region_boxes(...)##
获取检测到的boxes
void get_region_boxes(layer l, int w, int h, float thresh, float **probs, box *boxes, int only_objectness, int *map, float tree_thresh, int nomult)
{
int i,j,n,z;
float *predictions = l.output;
if (l.batch == 2) {
float *flip = l.output + l.outputs;
for (j = 0; j < l.h; ++j) {
for (i = 0; i < l.w/2; ++i) {
for (n = 0; n < l.n; ++n) {
for(z = 0; z < l.classes + 5; ++z){
int i1 = z*l.w*l.h*l.n + n*l.w*l.h + j*l.w + i;
int i2 = z*l.w*l.h*l.n + n*l.w*l.h + j*l.w + (l.w - i - 1);
float swap = flip[i1];
flip[i1] = flip[i2];
flip[i2] = swap;
if(z == 0){
flip[i1] = -flip[i1];
flip[i2] = -flip[i2];
}
}
}
}
}
for(i = 0; i < l.outputs; ++i){
l.output[i] = (l.output[i] + flip[i])/2.;
}
}
for (i = 0; i < l.w*l.h; ++i){
int row = i / l.w;
int col = i % l.w;
for(n = 0; n < l.n; ++n){
int index = n*l.w*l.h + i;
for(j = 0; j < l.classes; ++j){
probs[index][j] = 0;
}
int obj_index = entry_index(l, 0, n*l.w*l.h + i, 4);
int box_index = entry_index(l, 0, n*l.w*l.h + i, 0);
float scale = predictions[obj_index];
boxes[index] = get_region_box(predictions, l.biases, n, box_index, col, row, l.w, l.h, l.w*l.h);
if(1){
int max = w > h ? w : h;
boxes[index].x = (boxes[index].x - (max - w)/2./max) / ((float)w/max);
boxes[index].y = (boxes[index].y - (max - h)/2./max) / ((float)h/max);
boxes[index].w *= (float)max/w;
boxes[index].h *= (float)max/h;
}
if(!nomult){
boxes[index].x *= w;
boxes[index].y *= h;
boxes[index].w *= w;
boxes[index].h *= h;
}
int class_index = entry_index(l, 0, n*l.w*l.h + i, 5);
if(l.softmax_tree){
hierarchy_predictions(predictions + class_index, l.classes, l.softmax_tree, 0, l.w*l.h);
if(map){
for(j = 0; j < 200; ++j){
int class_index = entry_index(l, 0, n*l.w*l.h + i, 5 + map[j]);
float prob = scale*predictions[class_index];
probs[index][j] = (prob > thresh) ? prob : 0;
}
} else {
int j = hierarchy_top_prediction(predictions + class_index, l.softmax_tree, tree_thresh, l.w*l.h);
probs[index][j] = (scale > thresh) ? scale : 0;
probs[index][l.classes] = scale;
}
} else {
float max = 0;
for(j = 0; j < l.classes; ++j){
int class_index = entry_index(l, 0, n*l.w*l.h + i, 5 + j);
float prob = scale*predictions[class_index];
probs[index][j] = (prob > thresh) ? prob : 0;
if(prob > max) max = prob;
// TODO REMOVE
// if (j != 15 && j != 16) probs[index][j] = 0;
/*
if (j != 0) probs[index][j] = 0;
int blacklist[] = {121, 497, 482, 504, 122, 518,481, 418, 542, 491, 914, 478, 120, 510,500};
int bb;
for (bb = 0; bb < sizeof(blacklist)/sizeof(int); ++bb){
if(index == blacklist[bb]) probs[index][j] = 0;
}
*/
}
probs[index][l.classes] = max;
}
if(only_objectness){
probs[index][0] = scale;
}
}
}
}
void zero_objectness(...)##
这个函数在图像风格转换中用于把 object confidence 置为0
void zero_objectness(layer l)
{
int i, n;
for (i = 0; i < l.w*l.h; ++i){
for(n = 0; n < l.n; ++n){
int obj_index = entry_index(l, 0, n*l.w*l.h + i, 4);
l.output[obj_index] = 0;
}
}
}
