Pytorch-Faster-RCNN 中的 MAP 实现 （解析imdb.py 和 pascal_voc.py）

---恢复内容开始---

MAP是衡量object dectection算法的重要criteria，然而一直没有仔细阅读相关代码，今天就好好看一下：

1. 测试test过程是由FRCN/tools/test_net.py中调用的test_net()完成 #from model.test import test_net

test_net()定义在FRCN/lib/model/test.py （193-194行）：调用了imdb.evaluate_detections

print('Evaluating detections')
imdb.evaluate_detections(all_boxes, output_dir)

imdb是从FRCN/lib/model/test.py（84行）传入的：

imdb = get_imdb(args.imdb_name)

from datasets.factory import get_imdb，为了了解如何定义一个imdb，我们去FRCN/lib/datasets/factory.py

 """Factory method for easily getting imdbs by name."""
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

 __sets = {}
 from datasets.pascal_voc import pascal_voc

 import numpy as np

 # Set up voc_<year>_<split>
 for year in ['', '']:
   for split in ['train', 'val', 'trainval', 'test']:
     name = 'voc_{}_{}'.format(year, split)
     __sets[name] = (lambda split=split, year=year: pascal_voc(split, year))

 for year in ['', '']:
   for split in ['train', 'val', 'trainval', 'test']:
     name = 'voc_{}_{}_diff'.format(year, split)
     __sets[name] = (lambda split=split, year=year: pascal_voc(split, year, use_diff=True))

 def get_imdb(name):
   """Get an imdb (image database) by name."""
   if name not in __sets:
     raise KeyError('Unknown dataset: {}'.format(name))
   return __sets[name]()

 def list_imdbs():
   """List all registered imdbs."""
   return list(__sets.keys())

coco数据集的定义同pascal_voc. 可以看到，get_imdb(args.imdb_name)将会返回的就是pascal_voc(split, year)这样一个对象。

2. 来到pascal_voc.py ：

 # --------------------------------------------------------
 # Fast R-CNN
 # Copyright (c) 2015 Microsoft
 # Licensed under The MIT License [see LICENSE for details]
 # Written by Ross Girshick and Xinlei Chen
 # --------------------------------------------------------
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

 import os
 from datasets.imdb import imdb
 import datasets.ds_utils as ds_utils
 import xml.etree.ElementTree as ET
 import numpy as np
 import scipy.sparse
 import scipy.io as sio
 import pickle
 import subprocess
 import uuid
 from .voc_eval import voc_eval
 from model.config import cfg

 class pascal_voc(imdb):
   def __init__(self, image_set, year, use_diff=False):
     name = 'voc_' + year + '_' + image_set
     if use_diff:
       name += '_diff'
     imdb.__init__(self, name)
     self._year = year
     self._image_set = image_set
     self._devkit_path = self._get_default_path()
     self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year)
     self._classes = ('__background__',  # always index 0
                      'title', 'xlabel',  'ylabel')
                  ####    'text', 'ylabel')
                  #    'aeroplane', 'bicycle', 'bird', 'boat',
                  #    'bottle', 'bus', 'car', 'cat', 'chair',
                  #    'cow', 'diningtable', 'dog', 'horse',
                  #    'motorbike', 'person', 'pottedplant',
                  #    'sheep', 'sofa', 'train', 'tvmonitor')
     self._class_to_ind = dict(list(zip(self.classes, list(range(self.num_classes)))))
     self._image_ext = '.jpg'
     self._image_index = self._load_image_set_index()
     # Default to roidb handler
     self._roidb_handler = self.gt_roidb
     self._salt = str(uuid.uuid4())
     self._comp_id = 'comp4'

     # PASCAL specific config options
     self.config = {'cleanup': True,
                    'use_salt': True,
                    'use_diff': use_diff,
                    'matlab_eval': False,
                    'rpn_file': None}

     assert os.path.exists(self._devkit_path), \
       'VOCdevkit path does not exist: {}'.format(self._devkit_path)
     assert os.path.exists(self._data_path), \
       'Path does not exist: {}'.format(self._data_path)

   def image_path_at(self, i):
     """
     Return the absolute path to image i in the image sequence.
     """
     return self.image_path_from_index(self._image_index[i])

   def image_path_from_index(self, index):
     """
     Construct an image path from the image's "index" identifier.
     """
     image_path = os.path.join(self._data_path, 'JPEGImages',
                               index + self._image_ext)
     assert os.path.exists(image_path), \
       'Path does not exist: {}'.format(image_path)
     return image_path

   def _load_image_set_index(self):
     """
     Load the indexes listed in this dataset's image set file.
     """
     # Example path to image set file:
     # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt
     image_set_file = os.path.join(self._data_path, 'ImageSets', 'Main',
                                   self._image_set + '.txt')
     assert os.path.exists(image_set_file), \
       'Path does not exist: {}'.format(image_set_file)
     with open(image_set_file) as f:
       image_index = [x.strip() for x in f.readlines()]
     return image_index

   def _get_default_path(self):
     """
     Return the default path where PASCAL VOC is expected to be installed.
     """
     return os.path.join(cfg.DATA_DIR, 'VOCdevkit' + self._year)

   def gt_roidb(self):
     """
     Return the database of ground-truth regions of interest.

     This function loads/saves from/to a cache file to speed up future calls.
     """
     cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
     if os.path.exists(cache_file):
       with open(cache_file, 'rb') as fid:
         try:
           roidb = pickle.load(fid)
         except:
           roidb = pickle.load(fid, encoding='bytes')
       print('{} gt roidb loaded from {}'.format(self.name, cache_file))
       return roidb

     gt_roidb = [self._load_pascal_annotation(index)
                 for index in self.image_index]
     with open(cache_file, 'wb') as fid:
       pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
     print('wrote gt roidb to {}'.format(cache_file))

     return gt_roidb

   def rpn_roidb(self):
     if int(self._year) == 2007 or self._image_set != 'test':
       gt_roidb = self.gt_roidb()
       rpn_roidb = self._load_rpn_roidb(gt_roidb)
       roidb = imdb.merge_roidbs(gt_roidb, rpn_roidb)
     else:
       roidb = self._load_rpn_roidb(None)

     return roidb

   def _load_rpn_roidb(self, gt_roidb):
     filename = self.config['rpn_file']
     print('loading {}'.format(filename))
     assert os.path.exists(filename), \
       'rpn data not found at: {}'.format(filename)
     with open(filename, 'rb') as f:
       box_list = pickle.load(f)
     return self.create_roidb_from_box_list(box_list, gt_roidb)

   def _load_pascal_annotation(self, index):
     """
     Load image and bounding boxes info from XML file in the PASCAL VOC
     format.
     """
     filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
     tree = ET.parse(filename)
     objs = tree.findall('object')
     if not self.config['use_diff']:
       # Exclude the samples labeled as difficult
       non_diff_objs = [
         obj for obj in objs if int(obj.find('difficult').text) == 0]
       # if len(non_diff_objs) != len(objs):
       #     print 'Removed {} difficult objects'.format(
       #         len(objs) - len(non_diff_objs))
       objs = non_diff_objs
     num_objs = len(objs)

     boxes = np.zeros((num_objs, 4), dtype=np.uint16)
     gt_classes = np.zeros((num_objs), dtype=np.int32)
     overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)
     # "Seg" area for pascal is just the box area
     seg_areas = np.zeros((num_objs), dtype=np.float32)

     # Load object bounding boxes into a data frame.
     for ix, obj in enumerate(objs):
       bbox = obj.find('bndbox')
       # Make pixel indexes 0-based
       x1 = float(bbox.find('xmin').text) - 1
       y1 = float(bbox.find('ymin').text) - 1
       x2 = float(bbox.find('xmax').text) - 1
       y2 = float(bbox.find('ymax').text) - 1
       cls = self._class_to_ind[obj.find('name').text.lower().strip()]
       boxes[ix, :] = [x1, y1, x2, y2]
       gt_classes[ix] = cls
       overlaps[ix, cls] = 1.0
       seg_areas[ix] = (x2 - x1 + 1) * (y2 - y1 + 1)

     overlaps = scipy.sparse.csr_matrix(overlaps)

     return {'boxes': boxes,
             'gt_classes': gt_classes,
             'gt_overlaps': overlaps,
             'flipped': False,
             'seg_areas': seg_areas}

   def _get_comp_id(self):
     comp_id = (self._comp_id + '_' + self._salt if self.config['use_salt']
                else self._comp_id)
     return comp_id

   def _get_voc_results_file_template(self):
     # VOCdevkit/results/VOC2007/Main/<comp_id>_det_test_aeroplane.txt
     filename = self._get_comp_id() + '_det_' + self._image_set + '_{:s}.txt'
     path = os.path.join(
       self._devkit_path,
       'results',
       'VOC' + self._year,
       'Main',
       filename)
     return path

   def _write_voc_results_file(self, all_boxes):
     for cls_ind, cls in enumerate(self.classes):
       if cls == '__background__':
         continue
       print('Writing {} VOC results file'.format(cls))
       filename = self._get_voc_results_file_template().format(cls)
       with open(filename, 'wt') as f:
         for im_ind, index in enumerate(self.image_index):
           dets = all_boxes[cls_ind][im_ind]
           if dets == []:
             continue
           # the VOCdevkit expects 1-based indices
           for k in range(dets.shape[0]):
             f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'.
                     format(index, dets[k, -1],
                            dets[k, 0] + 1, dets[k, 1] + 1,
                            dets[k, 2] + 1, dets[k, 3] + 1))

   def _do_python_eval(self, output_dir='output'):
     annopath = os.path.join(
       self._devkit_path,
       'VOC' + self._year,
       'Annotations',
       '{:s}.xml')
     imagesetfile = os.path.join(
       self._devkit_path,
       'VOC' + self._year,
       'ImageSets',
       'Main',
       self._image_set + '.txt')
     cachedir = os.path.join(self._devkit_path, 'annotations_cache')
     aps = []
     # The PASCAL VOC metric changed in 2010
     use_07_metric = True if int(self._year) < 2010 else False
     print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
     if not os.path.isdir(output_dir):
       os.mkdir(output_dir)
     for i, cls in enumerate(self._classes):
       if cls == '__background__':
         continue
       filename = self._get_voc_results_file_template().format(cls)
       rec, prec, ap = voc_eval(
         filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5,
         use_07_metric=use_07_metric, use_diff=self.config['use_diff'])
       aps += [ap]
       print(('AP for {} = {:.4f}'.format(cls, ap)))
       with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
         pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
     print(('Mean AP = {:.4f}'.format(np.mean(aps))))
     print('~~~~~~~~')
     print('Results:')
     for ap in aps:
       print(('{:.3f}'.format(ap)))
     print(('{:.3f}'.format(np.mean(aps))))
     print('~~~~~~~~')
     print('')
     print('--------------------------------------------------------------')
     print('Results computed with the **unofficial** Python eval code.')
     print('Results should be very close to the official MATLAB eval code.')
     print('Recompute with `./tools/reval.py --matlab ...` for your paper.')
     print('-- Thanks, The Management')
     print('--------------------------------------------------------------')

   def _do_matlab_eval(self, output_dir='output'):
     print('-----------------------------------------------------')
     print('Computing results with the official MATLAB eval code.')
     print('-----------------------------------------------------')
     path = os.path.join(cfg.ROOT_DIR, 'lib', 'datasets',
                         'VOCdevkit-matlab-wrapper')
     cmd = 'cd {} && '.format(path)
     cmd += '{:s} -nodisplay -nodesktop '.format(cfg.MATLAB)
     cmd += '-r "dbstop if error; '
     cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\'); quit;"' \
       .format(self._devkit_path, self._get_comp_id(),
               self._image_set, output_dir)
     print(('Running:\n{}'.format(cmd)))
     status = subprocess.call(cmd, shell=True)

   def evaluate_detections(self, all_boxes, output_dir):
     self._write_voc_results_file(all_boxes)
     self._do_python_eval(output_dir)
     if self.config['matlab_eval']:
       self._do_matlab_eval(output_dir)
     if self.config['cleanup']:
       for cls in self._classes:
         if cls == '__background__':
           continue
         filename = self._get_voc_results_file_template().format(cls)
         os.remove(filename)

   def competition_mode(self, on):
     if on:
       self.config['use_salt'] = False
       self.config['cleanup'] = False
     else:
       self.config['use_salt'] = True
       self.config['cleanup'] = True

 if __name__ == '__main__':
   from datasets.pascal_voc import pascal_voc

   d = pascal_voc('trainval', '')
   res = d.roidb
   from IPython import embed;

   embed()

我们先看涉及到MAP的方法，其他方法暂时放下。

这里通过evaluate_detections方法调用了_do_python_eval方法，后者通过调用voc_eval函数进行了AP和MAP的计算（245-247行）。

 # --------------------------------------------------------
 # Fast/er R-CNN
 # Licensed under The MIT License [see LICENSE for details]
 # Written by Bharath Hariharan
 # --------------------------------------------------------
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

 import xml.etree.ElementTree as ET
 import os
 import pickle
 import numpy as np

 def parse_rec(filename):
   """ Parse a PASCAL VOC xml file """
   tree = ET.parse(filename)
   objects = []
   for obj in tree.findall('object'):
     obj_struct = {}
     obj_struct['name'] = obj.find('name').text
     obj_struct['pose'] = obj.find('pose').text
     obj_struct['truncated'] = int(obj.find('truncated').text)
     obj_struct['difficult'] = int(obj.find('difficult').text)
     bbox = obj.find('bndbox')
     obj_struct['bbox'] = [int(float(bbox.find('xmin').text)),
                           int(float(bbox.find('ymin').text)),
                           int(float(bbox.find('xmax').text)),
                           int(float(bbox.find('ymax').text))]
     objects.append(obj_struct)

   return objects

 def voc_ap(rec, prec, use_07_metric=False):
   """ ap = voc_ap(rec, prec, [use_07_metric])
   Compute VOC AP given precision and recall.
   If use_07_metric is true, uses the
   VOC 07 11 point method (default:False).
   """
   if use_07_metric:
     # 11 point metric
     ap = 0.
     for t in np.arange(0., 1.1, 0.1):
       if np.sum(rec >= t) == 0:
         p = 0
       else:
         p = np.max(prec[rec >= t])
       ap = ap + p / 11.
   else:
     # correct AP calculation
     # first append sentinel values at the end
     mrec = np.concatenate(([0.], rec, [1.]))
     mpre = np.concatenate(([0.], prec, [0.]))

     # compute the precision envelope
     for i in range(mpre.size - 1, 0, -1):
       mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])

     # to calculate area under PR curve, look for points
     # where X axis (recall) changes value
     i = np.where(mrec[1:] != mrec[:-1])[0]

     # and sum (\Delta recall) * prec
     ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
   return ap

 def voc_eval(detpath,
              annopath,
              imagesetfile,
              classname,
              cachedir,
              ovthresh=0.5,
              use_07_metric=False,
              use_diff=False):
   """rec, prec, ap = voc_eval(detpath,
                               annopath,
                               imagesetfile,
                               classname,
                               [ovthresh],
                               [use_07_metric])

   Top level function that does the PASCAL VOC evaluation.

   detpath: Path to detections
       detpath.format(classname) should produce the detection results file.
   annopath: Path to annotations
       annopath.format(imagename) should be the xml annotations file.
   imagesetfile: Text file containing the list of images, one image per line.
   classname: Category name (duh)
   cachedir: Directory for caching the annotations
   [ovthresh]: Overlap threshold (default = 0.5)
   [use_07_metric]: Whether to use VOC07's 11 point AP computation
       (default False)
   """
   # assumes detections are in detpath.format(classname)
   # assumes annotations are in annopath.format(imagename)
   # assumes imagesetfile is a text file with each line an image name
   # cachedir caches the annotations in a pickle file

   # first load gt
   if not os.path.isdir(cachedir):
     os.mkdir(cachedir)
   cachefile = os.path.join(cachedir, '%s_annots.pkl' % imagesetfile)
   # read list of images
   with open(imagesetfile, 'r') as f:
     lines = f.readlines()
   imagenames = [x.strip() for x in lines]    #test.txt中的所有标号

   # load annotations
   if not os.path.isfile(cachefile):
     recs = {}
     for i, imagename in enumerate(imagenames):
       recs[imagename] = parse_rec(annopath.format(imagename))
       if i % 100 == 0:
         print('Reading annotation for {:d}/{:d}'.format(
           i + 1, len(imagenames)))
     # save
     print('Saving cached annotations to {:s}'.format(cachefile))
     with open(cachefile, 'wb') as f:
       pickle.dump(recs, f)
   else:
     # load
     with open(cachefile, 'rb') as f:
       try:
         recs = pickle.load(f)
       except:
         recs = pickle.load(f, encoding='bytes')

   # extract gt objects for this class
   class_recs = {}
   npos = 0
   for imagename in imagenames:
     R = [obj for obj in recs[imagename] if obj['name'] == classname]
     bbox = np.array([x['bbox'] for x in R])
     if use_diff:
       difficult = np.array([False for x in R]).astype(np.bool)
     else:
       difficult = np.array([x['difficult'] for x in R]).astype(np.bool)
     det = [False] * len(R)
     npos = npos + sum(~difficult)
     class_recs[imagename] = {'bbox': bbox,
                              'difficult': difficult,
                              'det': det}

   # read dets
   detfile = detpath.format(classname)
   with open(detfile, 'r') as f:
     lines = f.readlines()

   splitlines = [x.strip().split(' ') for x in lines]
   image_ids = [x[0] for x in splitlines]
   confidence = np.array([float(x[1]) for x in splitlines])
   BB = np.array([[float(z) for z in x[2:]] for x in splitlines])

   nd = len(image_ids)
   tp = np.zeros(nd)
   fp = np.zeros(nd)

   if BB.shape[0] > 0:
     # sort by confidence
     sorted_ind = np.argsort(-confidence)
     sorted_scores = np.sort(-confidence)
     BB = BB[sorted_ind, :]
     image_ids = [image_ids[x] for x in sorted_ind]

     # go down dets and mark TPs and FPs
     for d in range(nd):
       R = class_recs[image_ids[d]]
       bb = BB[d, :].astype(float)
       ovmax = -np.inf
       BBGT = R['bbox'].astype(float)

       if BBGT.size > 0:
         # compute overlaps
         # intersection
         ixmin = np.maximum(BBGT[:, 0], bb[0])
         iymin = np.maximum(BBGT[:, 1], bb[1])
         ixmax = np.minimum(BBGT[:, 2], bb[2])
         iymax = np.minimum(BBGT[:, 3], bb[3])
         iw = np.maximum(ixmax - ixmin + 1., 0.)
         ih = np.maximum(iymax - iymin + 1., 0.)
         inters = iw * ih

         # union
         uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                (BBGT[:, 2] - BBGT[:, 0] + 1.) *
                (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)

         overlaps = inters / uni
         ovmax = np.max(overlaps)
         jmax = np.argmax(overlaps)

       if ovmax > ovthresh:
         if not R['difficult'][jmax]:
           if not R['det'][jmax]:
             tp[d] = 1.
             R['det'][jmax] = 1
           else:
             fp[d] = 1.
       else:
         fp[d] = 1.

   # compute precision recall
   fp = np.cumsum(fp)
   tp = np.cumsum(tp)
   rec = tp / float(npos)
   # avoid divide by zero in case the first detection matches a difficult
   # ground truth
   prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
   ap = voc_ap(rec, prec, use_07_metric)

   return rec, prec, ap

voc_eval(filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5, use_07_metric=use_07_metric, use_diff=self.config['use_diff'])

def voc_eval(detpath, annopath, imagesetfile, classname, cachedir, ovthresh=0.5, use_07_metric=False, use_diff=False):

filename: detpath: Path to detections 存储detection结果的pkl文件地址

annopath: 存储Annotations的地址

imagesetfile: 图片集的txt文档

classname: 当前的class

cachedir: 存储Annotations的pkl所在目录（可能不存在）

ovthresh=0.5: IoU的threshold，默认为0.5

use_07_metric=Flase: 是否使用2007PASCAL_VOC的MAP计算规则

use_diff=False: 是否考虑difficult的检测样本

经过一番数据处理，得到了：

BB： 当前class的所有proposal bbox （predicted）

image_ids: 当前imageset的所有image序号

class_recs: image所包含的当前class的bbox （GT）

   if BB.shape[0] > 0:
     # sort by confidence
     #'''
     sorted_ind = np.argsort(-confidence)
     sorted_scores = np.sort(-confidence)
     BB = BB[sorted_ind, :]    # 现在的BB是按照conf降序排列的所有predicted bbox
     image_ids = [image_ids[x] for x in sorted_ind]    # image_id 是BB每组bbox所属于的image的序号

     #'''

     # go down dets and mark TPs and FPs
     for d in range(nd):              #对所有proposal bbox 遍历
       R = class_recs[image_ids[d]]   # 找到当前bbox对应的image
       bb = BB[d, :].astype(float)    # bb 为当前proposal bbox的坐标
       ovmax = -np.inf                # 设置np极小值
       BBGT = R['bbox'].astype(float) 

       if BBGT.size > 0:
         # compute overlaps
         # intersection
         ixmin = np.maximum(BBGT[:, 0], bb[0])
         iymin = np.maximum(BBGT[:, 1], bb[1])
         ixmax = np.minimum(BBGT[:, 2], bb[2])
         iymax = np.minimum(BBGT[:, 3], bb[3])
         iw = np.maximum(ixmax - ixmin + 1., 0.)
         ih = np.maximum(iymax - iymin + 1., 0.)
         inters = iw * ih

         # union
         uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
                (BBGT[:, 2] - BBGT[:, 0] + 1.) *
                (BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)

         overlaps = inters / uni
         ovmax = np.max(overlaps)
         jmax = np.argmax(overlaps)
         print(overlaps)

       if ovmax > ovthresh:
         if not R['difficult'][jmax]:
           if not R['det'][jmax]:        #是否已经被检测过
             tp[d] = 1.
             R['det'][jmax] = 1
           else:
             fp[d] = 1.
       else:
         fp[d] = 1.

疑惑：

这里的Recall计算（voc_eval.py 208行）使用了：

rec = tp / float(npos)，npos实际上是所有bbox-GT的数量，并不应该等于tp+fn吧？当且仅当：fn（包含但未被检测出bbox的image数量）==npos-tp（未被检测出的bbox数量）

ref: 1. https://datascience.stackexchange.com/questions/25119/how-to-calculate-map-for-detection-task-for-the-pascal-voc-challenge

2. http://mp.weixin.qq.com/s/FaNC9RppIhPf6T_qAz3Slg

3. https://ils.unc.edu/courses/2013_spring/inls509_001/lectures/10-EvaluationMetrics.pdf

4. https://stats.stackexchange.com/questions/260430/average-precision-in-object-detection/263758#263758