OpenCV245之SURF源代码分析

一、fastHessianDetector函数分析

（1）參数

const Mat& sum                积分图片

const Mat& mask_sum

vector<KeyPoint>& keypoints   关键点

int nOctaves                  金字塔的阶数

int nOctaveLayers             每阶金字塔的中间层数

float hessianThreshold        Hessian阀值

（2）函数体

static void fastHessianDetector( const Mat& sum, const Mat& mask_sum, vector<KeyPoint>& keypoints,
                                 int nOctaves, int nOctaveLayers, float hessianThreshold )
{
    const int SAMPLE_STEP0 = 1;

    int nTotalLayers = (nOctaveLayers+2)*nOctaves;//全部层的数目
    int nMiddleLayers = nOctaveLayers*nOctaves;//全部中间层的数目

    vector<Mat> dets(nTotalLayers);//全部层dets
    vector<Mat> traces(nTotalLayers);//全部层tra
    vector<int> sizes(nTotalLayers);//全部层滤波器尺寸
    vector<int> sampleSteps(nTotalLayers);//全部层採样比例
    vector<int> middleIndices(nMiddleLayers);//全部中间层的索引

    keypoints.clear();

    // Allocate space and calculate properties of each layer
    int index = 0, middleIndex = 0, step = SAMPLE_STEP0;//第一阶梯的採样比例设置为1

    for( int octave = 0; octave < nOctaves; octave++ )//阶梯循环
    {
        for( int layer = 0; layer < nOctaveLayers+2; layer++ )//层循环
        {
            //申请内存
            dets[index].create( (sum.rows-1)/step, (sum.cols-1)/step, CV_32F );
            traces[index].create( (sum.rows-1)/step, (sum.cols-1)/step, CV_32F );
            sizes[index] = (SURF_HAAR_SIZE0 + SURF_HAAR_SIZE_INC*layer) << octave;//滤波器大小
            sampleSteps[index] = step;//图片比例尺寸

            if( 0 < layer && layer <= nOctaveLayers )//记录中间层索引
                middleIndices[middleIndex++] = index;
            index++;
        }
        step *= 2;//进入下一阶梯时。比例放大两倍
    }

    //生成全部层数据。SURFBuildInvoker函数在第（3）点分析
    parallel_for( BlockedRange(0, nTotalLayers),
                      SURFBuildInvoker(sum, sizes, sampleSteps, dets, traces) );

    // Find maxima in the determinant of the hessian
    parallel_for( BlockedRange(0, nMiddleLayers),
                      SURFFindInvoker(sum, mask_sum, dets, traces, sizes,
                                      sampleSteps, middleIndices, keypoints,
                                      nOctaveLayers, hessianThreshold) );

    std::sort(keypoints.begin(), keypoints.end(), KeypointGreater());
}  

（3）SURFBuildInvoker

构建每一阶各层的数据，详细函数calcLayerDetAndTrace看第（4）点

struct SURFBuildInvoker
{
    SURFBuildInvoker( const Mat& _sum, const vector<int>& _sizes,
                      const vector<int>& _sampleSteps,
                      vector<Mat>& _dets, vector<Mat>& _traces )
    {
        sum = &_sum;
        sizes = &_sizes;
        sampleSteps = &_sampleSteps;
        dets = &_dets;
        traces = &_traces;
    }

    void operator()(const BlockedRange& range) const
    {
        for( int i=range.begin(); i<range.end(); i++ )
            calcLayerDetAndTrace( *sum, (*sizes)[i], (*sampleSteps)[i], (*dets)[i], (*traces)[i] );
    }

    const Mat *sum;
    const vector<int> *sizes;
    const vector<int> *sampleSteps;
    vector<Mat>* dets;
    vector<Mat>* traces;
};

（4）calcLayerDetAndTrace

功能：计算每一层的数据

參数：

const Mat& sum    积分图像

int size                    滤波器大小

sampleStep            图像採样比例

Mat& det                det数据

Mat& trace            trace数据

函数体：

static void calcLayerDetAndTrace( const Mat& sum, int size, int sampleStep,
                                  Mat& det, Mat& trace )
{
    const int NX=3, NY=3, NXY=4;//3个滤波器的各个子块
    //滤波器各个子块在起始位置的X起始坐标，Y起始坐标，X结束坐标。Y结束坐标。以及加权数
    const int dx_s[NX][5] = { {0, 2, 3, 7, 1}, {3, 2, 6, 7, -2}, {6, 2, 9, 7, 1} };
    const int dy_s[NY][5] = { {2, 0, 7, 3, 1}, {2, 3, 7, 6, -2}, {2, 6, 7, 9, 1} };
    const int dxy_s[NXY][5] = { {1, 1, 4, 4, 1}, {5, 1, 8, 4, -1}, {1, 5, 4, 8, -1}, {5, 5, 8, 8, 1} };

    /*
    SurfHF结构体：滤波器各个子块4个坐标
    struct SurfHF
    {
        int p0, p1, p2, p3;
        float w;

        SurfHF(): p0(0), p1(0), p2(0), p3(0), w(0) {}
    };
    */
    SurfHF Dx[NX], Dy[NY], Dxy[NXY];

    if( size > sum.rows-1 || size > sum.cols-1 )
       return;

    //计算出每一层滤波器各个子块在原积分图像中的起始坐标，在第（5）点分析
    resizeHaarPattern( dx_s , Dx , NX , 9, size, sum.cols );
    resizeHaarPattern( dy_s , Dy , NY , 9, size, sum.cols );
    resizeHaarPattern( dxy_s, Dxy, NXY, 9, size, sum.cols );

    /* The integral image 'sum' is one pixel bigger than the source image */
    //在sampleStep比例下计算的长宽
    //减去滤波器大小size防止越界，之所以是size而不是size/2,是由于把起始与结束的越界范围一起算进来
    int samples_i = 1+(sum.rows-1-size)/sampleStep;
    int samples_j = 1+(sum.cols-1-size)/sampleStep;

    /* Ignore pixels where some of the kernel is outside the image */
    //计算在缩小sampleStep比例后得越界大小
    int margin = (size/2)/sampleStep;

    for( int i = 0; i < samples_i; i++ )
    {
        //获取原积分图像在第i*sampleStep行的指针
        const int* sum_ptr = sum.ptr<int>(i*sampleStep);
        //获取det、trace在第i+margin行、第margin个的数据指针
        float* det_ptr = &det.at<float>(i+margin, margin);
        float* trace_ptr = &trace.at<float>(i+margin, margin);
        for( int j = 0; j < samples_j; j++ )
        {
            //计算滤波器子块所覆盖区域的加权后的积分值。在第（6）点分析
            float dx  = calcHaarPattern( sum_ptr, Dx , 3 );
            float dy  = calcHaarPattern( sum_ptr, Dy , 3 );
            float dxy = calcHaarPattern( sum_ptr, Dxy, 4 );
            //调至原积分图像中要计算下一个坐标点
            sum_ptr += sampleStep;
            //计算det_ptr、trace_ptr
            det_ptr[j] = dx*dy - 0.81f*dxy*dxy;
            trace_ptr[j] = dx + dy;
        }
    }
}

（5）resizeHaarPattern

功能：生成新滤波器子块在原图像中的坐标点

參数：

const int src[][5]

SurfHF* dst

int n                      滤波器子块数目

int oldSize              旧滤波器大小

int newSize            新滤波器大小

int widthStep         原图像宽度

函数体：

static void
resizeHaarPattern( const int src[][5], SurfHF* dst, int n, int oldSize, int newSize, int widthStep )
{
    float ratio = (float)newSize/oldSize;
    for( int k = 0; k < n; k++ )
    {
        //计算新滤波器子块4个点坐标
        int dx1 = cvRound( ratio*src[k][0] );//四舍五入
        int dy1 = cvRound( ratio*src[k][1] );
        int dx2 = cvRound( ratio*src[k][2] );
        int dy2 = cvRound( ratio*src[k][3] );
        //计算4个点在原图像中的坐标
        dst[k].p0 = dy1*widthStep + dx1;
        dst[k].p1 = dy2*widthStep + dx1;
        dst[k].p2 = dy1*widthStep + dx2;
        dst[k].p3 = dy2*widthStep + dx2;
        //src[k][4]为加权数，1.0 /((float)(dx2-dx1)*(dy2-dy1))求此子块的均值
        dst[k].w = src[k][4]/((float)(dx2-dx1)*(dy2-dy1));
    }
}

（6）calcHaarPattern

功能：计算滤波器子块所覆盖区域的加权后的积分值

參数：

const int* origin  原积分图像要计算的坐标点指针

const SurfHF* f    resizeHaarPattern中计算出的坐标定位点。或者说滤波器子块的相对于中心点的坐标关系

int n                     子块数目

函数体：

inline float calcHaarPattern( const int* origin, const SurfHF* f, int n )
{
    double d = 0;
    for( int k = 0; k < n; k++ )
        d += (origin[f[k].p0] + origin[f[k].p3] - origin[f[k].p1] - origin[f[k].p2])*f[k].w;
    return (float)d;
}

分析完SURFBuildInvoker以及其各个子函数，如今以下開始分析利用上面得到的金字塔数据来进行非极大值抑制

（7）SURFFindInvoker

功能：金字塔数据来进行非极大值抑制，找到极值点并插值

findMaximaInLayer在第（8）点分析

參数：

const Mat& _sum          原积分图数据

const Mat& _mask_sum,

const vector<Mat>& _dets

const vector<Mat>& _traces

const vector<int>& _sizes        滤波器大小

const vector<int>& _sampleSteps      图片比例尺寸

const vector<int>& _middleIndices  中间层索引

vector<KeyPoint>& _keypoints    关键点

int _nOctaveLayers          每一阶中间层数目

float _hessianThreshold             hessian阀值

函数体：

struct SURFFindInvoker
{
    SURFFindInvoker( const Mat& _sum, const Mat& _mask_sum,
                     const vector<Mat>& _dets, const vector<Mat>& _traces,
                     const vector<int>& _sizes, const vector<int>& _sampleSteps,
                     const vector<int>& _middleIndices, vector<KeyPoint>& _keypoints,
                     int _nOctaveLayers, float _hessianThreshold )
    {
        sum = &_sum;
        mask_sum = &_mask_sum;
        dets = &_dets;
        traces = &_traces;
        sizes = &_sizes;
        sampleSteps = &_sampleSteps;
        middleIndices = &_middleIndices;
        keypoints = &_keypoints;
        nOctaveLayers = _nOctaveLayers;
        hessianThreshold = _hessianThreshold;
    }
    //在第（8）点分析
    static void findMaximaInLayer( const Mat& sum, const Mat& mask_sum,
                   const vector<Mat>& dets, const vector<Mat>& traces,
                   const vector<int>& sizes, vector<KeyPoint>& keypoints,
                   int octave, int layer, float hessianThreshold, int sampleStep );

    void operator()(const BlockedRange& range) const
    {
        for( int i=range.begin(); i<range.end(); i++ )
        {
            int layer = (*middleIndices)[i];
            int octave = i / nOctaveLayers;
            findMaximaInLayer( *sum, *mask_sum, *dets, *traces, *sizes,
                               *keypoints, octave, layer, hessianThreshold,
                               (*sampleSteps)[layer] );
        }
    }

    const Mat *sum;
    const Mat *mask_sum;
    const vector<Mat>* dets;
    const vector<Mat>* traces;
    const vector<int>* sizes;
    const vector<int>* sampleSteps;
    const vector<int>* middleIndices;
    vector<KeyPoint>* keypoints;
    int nOctaveLayers;
    float hessianThreshold;

#ifdef HAVE_TBB
    static tbb::mutex findMaximaInLayer_m;
#endif
};

（8）findMaximaInLayer

功能：确定是否为3X3X3中的极大值

參数：

const Mat& sum         原积分图像

const Mat& mask_sum

const vector<Mat>& dets

const vector<Mat>& traces

const vector<int>& sizes    滤波器大小数组

vector<KeyPoint>& keypoints

int octave                   中间层所在的阶

int layer                   中间层索引

float hessianThreshold      hessian阀值

int sampleStep               所在中间层的图像比例

函数体：

void SURFFindInvoker::findMaximaInLayer( const Mat& sum, const Mat& mask_sum,
                   const vector<Mat>& dets, const vector<Mat>& traces,
                   const vector<int>& sizes, vector<KeyPoint>& keypoints,
                   int octave, int layer, float hessianThreshold, int sampleStep )
{
    // Wavelet Data
    const int NM=1;
    const int dm[NM][5] = { {0, 0, 9, 9, 1} };
    SurfHF Dm;

    int size = sizes[layer];//获取所在层的滤波器大小

    // The integral image 'sum' is one pixel bigger than the source image
    //计算缩小sampleStep比例后的图像的cols,rows
    int layer_rows = (sum.rows-1)/sampleStep;
    int layer_cols = (sum.cols-1)/sampleStep;

    // Ignore pixels without a 3x3x3 neighbourhood in the layer above
    //通过最顶层滤波器所在越界区域大小。由于最顶层滤波器在比較的3层中是最大的
    int margin = (sizes[layer+1]/2)/sampleStep+1;

    if( !mask_sum.empty() )
       resizeHaarPattern( dm, &Dm, NM, 9, size, mask_sum.cols );

    int step = (int)(dets[layer].step/dets[layer].elemSize());//获取所在层的像素宽

    for( int i = margin; i < layer_rows - margin; i++ )
    {
        const float* det_ptr = dets[layer].ptr<float>(i);
        const float* trace_ptr = traces[layer].ptr<float>(i);
        for( int j = margin; j < layer_cols-margin; j++ )
        {
            float val0 = det_ptr[j];
            if( val0 > hessianThreshold )//剔除对照度低的点
            {
                /* Coordinates for the start of the wavelet in the sum image. There
                   is some integer division involved, so don't try to simplify this
                   (cancel out sampleStep) without checking the result is the same */
                //计算小波变换在原积分图像中的起始坐标
                int sum_i = sampleStep*(i-(size/2)/sampleStep);
                int sum_j = sampleStep*(j-(size/2)/sampleStep);

                /* The 3x3x3 neighbouring samples around the maxima.
                   The maxima is included at N9[1][4] */

                const float *det1 = &dets[layer-1].at<float>(i, j);//比較的最底层
                const float *det2 = &dets[layer].at<float>(i, j);//比較的中间层
                const float *det3 = &dets[layer+1].at<float>(i, j);//比較的最顶层
                //获取要比較的26个点数据
                float N9[3][9] = { { det1[-step-1], det1[-step], det1[-step+1],
                                     det1[-1]  , det1[0] , det1[1],
                                     det1[step-1] , det1[step] , det1[step+1]  },
                                   { det2[-step-1], det2[-step], det2[-step+1],
                                     det2[-1]  , det2[0] , det2[1],
                                     det2[step-1] , det2[step] , det2[step+1]  },
                                   { det3[-step-1], det3[-step], det3[-step+1],
                                     det3[-1]  , det3[0] , det3[1],
                                     det3[step-1] , det3[step] , det3[step+1]  } };

                /* Check the mask - why not just check the mask at the center of the wavelet? */
                if( !mask_sum.empty() )
                {
                    const int* mask_ptr = &mask_sum.at<int>(sum_i, sum_j);
                    float mval = calcHaarPattern( mask_ptr, &Dm, 1 );
                    if( mval < 0.5 )
                        continue;
                }//!mask_sum.empty()

                /* Non-maxima suppression. val0 is at N9[1][4]*/
                //比較val0是不是3X3区域内的极大值
                if( val0 > N9[0][0] && val0 > N9[0][1] && val0 > N9[0][2] &&
                    val0 > N9[0][3] && val0 > N9[0][4] && val0 > N9[0][5] &&
                    val0 > N9[0][6] && val0 > N9[0][7] && val0 > N9[0][8] &&
                    val0 > N9[1][0] && val0 > N9[1][1] && val0 > N9[1][2] &&
                    val0 > N9[1][3]                    && val0 > N9[1][5] &&
                    val0 > N9[1][6] && val0 > N9[1][7] && val0 > N9[1][8] &&
                    val0 > N9[2][0] && val0 > N9[2][1] && val0 > N9[2][2] &&
                    val0 > N9[2][3] && val0 > N9[2][4] && val0 > N9[2][5] &&
                    val0 > N9[2][6] && val0 > N9[2][7] && val0 > N9[2][8] )
                {
                    /* Calculate the wavelet center coordinates for the maxima */
                    //获取小波变化的中心坐标
                    float center_i = sum_i + (size-1)*0.5f;//y
                    float center_j = sum_j + (size-1)*0.5f;//x

                    /*
                    kpt.pt.x = center_j;//关键点在原图像中的坐标
                    kpt.pt.y = center_i;
                    kpt.size = (float)sizes[layer]//获取所在层的滤波器大小
                    kpt.angle = -1;//方向未定
                    kpt.response = val0;//海參响应值
                    kpt.class_id = CV_SIGN(trace_ptr[j]);//CV_SIGN(trace_ptr[j])推断dx+dy是否大于0,是为1，不是为-1
                    */
                    KeyPoint kpt( center_j, center_i, (float)sizes[layer],
                                  -1, val0, octave, CV_SIGN(trace_ptr[j]) );

                    /* Interpolate maxima location within the 3x3x3 neighbourhood  */
                    int ds = size - sizes[layer-1];//中间层与底层的滤波器尺寸差
                    //插值。在第（9）点分析
                    int interp_ok = interpolateKeypoint( N9, sampleStep, sampleStep, ds, kpt );

                    /* Sometimes the interpolation step gives a negative size etc. */
                    if( interp_ok  )
                    {
                        /*printf( "KeyPoint %f %f %d\n", point.pt.x, point.pt.y, point.size );*/
//#ifdef HAVE_TBB
//                        tbb::mutex::scoped_lock lock(findMaximaInLayer_m);
//#endif
                        keypoints.push_back(kpt);
                    }//interp_ok
                }//Non-maxima suppression
            }//val0 > hessianThreshold
        }//j
    }//i
}

（9）interpolateKeypoint

功能：进行像素点插值，并推断是否合格

參数：

float N9[3][9] 3X3X3个数据

int dx         X方向上的图像比例

int dy
   Y方向上的图像比例

int ds         与邻近层的滤波器尺寸差

KeyPoint& kpt  关键点

函数体：

static int
interpolateKeypoint( float N9[3][9], int dx, int dy, int ds, KeyPoint& kpt )
{
    // 像素在x。y，s三个方向的偏导数
    Vec3f b(-(N9[1][5]-N9[1][3])/2,  // Negative 1st deriv with respect to x
            -(N9[1][7]-N9[1][1])/2,  // Negative 1st deriv with respect to y
            -(N9[2][4]-N9[0][4])/2); // Negative 1st deriv with respect to s

    // 像素的Hessian矩阵
    Matx33f A(
        N9[1][3]-2*N9[1][4]+N9[1][5],            // 2nd deriv x, x
        (N9[1][8]-N9[1][6]-N9[1][2]+N9[1][0])/4, // 2nd deriv x, y
        (N9[2][5]-N9[2][3]-N9[0][5]+N9[0][3])/4, // 2nd deriv x, s
        (N9[1][8]-N9[1][6]-N9[1][2]+N9[1][0])/4, // 2nd deriv x, y
        N9[1][1]-2*N9[1][4]+N9[1][7],            // 2nd deriv y, y
        (N9[2][7]-N9[2][1]-N9[0][7]+N9[0][1])/4, // 2nd deriv y, s
        (N9[2][5]-N9[2][3]-N9[0][5]+N9[0][3])/4, // 2nd deriv x, s
        (N9[2][7]-N9[2][1]-N9[0][7]+N9[0][1])/4, // 2nd deriv y, s
        N9[0][4]-2*N9[1][4]+N9[2][4]);           // 2nd deriv s, s

    Vec3f x = A.solve(b, DECOMP_LU);

    bool ok = (x[0] != 0 || x[1] != 0 || x[2] != 0) &&
        std::abs(x[0]) <= 1 && std::abs(x[1]) <= 1 && std::abs(x[2]) <= 1;

    if( ok )
    {
        //计算插值后的坐标与滤波器尺寸
        kpt.pt.x += x[0]*dx;
        kpt.pt.y += x[1]*dy;
        kpt.size = (float)cvRound( kpt.size + x[2]*ds );
    }
    return ok;
}

SURF的金字塔构建、局部极大值提取至此完毕。以下是候选点的描写叙述符提取算法分析。

二、SURFInvoker

參数：

const Mat& _img

const Mat& _sum
 原积分图像

vector<KeyPoint>& _keypoints 关键点

Mat& _descriptors            描写叙述符

bool _extended               是否把64维描写叙述符拓展到128维

bool _upright                是否考虑旋转,false表示要考虑，upright为垂直的意思

函数体：

struct SURFInvoker
{
    enum { ORI_RADIUS = 6, ORI_WIN = 60, PATCH_SZ = 20 };

    SURFInvoker( const Mat& _img, const Mat& _sum,
                 vector<KeyPoint>& _keypoints, Mat& _descriptors,
                 bool _extended, bool _upright )
    {
        keypoints = &_keypoints;
        descriptors = &_descriptors;
        img = &_img;
        sum = &_sum;
        extended = _extended;
        upright = _upright;

        // Simple bound for number of grid points in circle of radius ORI_RADIUS
        const int nOriSampleBound = (2*ORI_RADIUS+1)*(2*ORI_RADIUS+1);

        // Allocate arrays
        apt.resize(nOriSampleBound);// 特征点周围用于描写叙述方向的邻域的点
        aptw.resize(nOriSampleBound);// 描写叙述方向时的高斯权
        //以20s为边长的矩形窗体为特征描写叙述子计算使用的窗体
        DW.resize(PATCH_SZ*PATCH_SZ); //特征描写叙述子的高斯加权。PATHC_SZ为特征描写叙述子的区域大小 20s(s这里初始为1了)

        /* Coordinates and weights of samples used to calculate orientation */
        //生成高斯模板，用于描写叙述方向时的高斯权
        //（s=1.2∗L/9为特征点的尺度）这里把s近似为1 ORI_DADIUS = 6s
        //nOriSampleBound为 矩形框内点的个数
        Mat G_ori = getGaussianKernel( 2*ORI_RADIUS+1, SURF_ORI_SIGMA, CV_32F );
        nOriSamples = 0;
        for( int i = -ORI_RADIUS; i <= ORI_RADIUS; i++ )
        {
            for( int j = -ORI_RADIUS; j <= ORI_RADIUS; j++ )
            {
                if( i*i + j*j <= ORI_RADIUS*ORI_RADIUS )
                {
                    apt[nOriSamples] = cvPoint(i,j);
                    aptw[nOriSamples++] = G_ori.at<float>(i+ORI_RADIUS,0) * G_ori.at<float>(j+ORI_RADIUS,0);
                }
            }
        }
        CV_Assert( nOriSamples <= nOriSampleBound );

        /* Gaussian used to weight descriptor samples */
         //用于生成特征描写叙述子的 高斯加权 sigma = 3.3s(s初取1)
        Mat G_desc = getGaussianKernel( PATCH_SZ, SURF_DESC_SIGMA, CV_32F );
        for( int i = 0; i < PATCH_SZ; i++ )
        {
            for( int j = 0; j < PATCH_SZ; j++ )
                DW[i*PATCH_SZ+j] = G_desc.at<float>(i,0) * G_desc.at<float>(j,0);
        }
    }

    void operator()(const BlockedRange& range) const
    {
        /* X and Y gradient wavelet data */
         /* x与y方向上的 Harr小波,參数为4s */
        const int NX=2, NY=2;
        const int dx_s[NX][5] = {{0, 0, 2, 4, -1}, {2, 0, 4, 4, 1}};
        const int dy_s[NY][5] = {{0, 0, 4, 2, 1}, {0, 2, 4, 4, -1}};

        // Optimisation is better using nOriSampleBound than nOriSamples for
        // array lengths.  Maybe because it is a constant known at compile time
        const int nOriSampleBound =(2*ORI_RADIUS+1)*(2*ORI_RADIUS+1);

        //用于计算HarrX响应、HarrY响应、特征点主方向
        float X[nOriSampleBound], Y[nOriSampleBound], angle[nOriSampleBound];
        uchar PATCH[PATCH_SZ+1][PATCH_SZ+1];
        // 20s * 20s区域的 梯度值
        float DX[PATCH_SZ][PATCH_SZ], DY[PATCH_SZ][PATCH_SZ];

        CvMat matX = cvMat(1, nOriSampleBound, CV_32F, X);
        CvMat matY = cvMat(1, nOriSampleBound, CV_32F, Y);
        CvMat _angle = cvMat(1, nOriSampleBound, CV_32F, angle);
        Mat _patch(PATCH_SZ+1, PATCH_SZ+1, CV_8U, PATCH);

        int dsize = extended ?

128 : 64;//是否把64维描写叙述符拓展到128维

        int k, k1 = range.begin(), k2 = range.end();//keypoints数目
        float maxSize = 0;
        for( k = k1; k < k2; k++ )
        {
            //找出关键点中Harr小波模板最大尺寸
            maxSize = std::max(maxSize, (*keypoints)[k].size);
        }

         // maxSize*1.2/9 表示最大的尺度 s
        int imaxSize = std::max(cvCeil((PATCH_SZ+1)*maxSize*1.2f/9.0f), 1);
        //用于20*s邻域窗体缓存
        Ptr<CvMat> winbuf = cvCreateMat( 1, imaxSize*imaxSize, CV_8U );

        for( k = k1; k < k2; k++ )
        {
            int i, j, kk, nangle;
            float* vec;
            SurfHF dx_t[NX], dy_t[NY];
            KeyPoint& kp = (*keypoints)[k];
            float size = kp.size;//此关键点的Harr小波模板尺寸
            Point2f center = kp.pt;//获取关键点坐标，座位模板中心点
            /* The sampling intervals and wavelet sized for selecting an orientation
             and building the keypoint descriptor are defined relative to 's' */
            float s = size*1.2f/9.0f;
            /* To find the dominant orientation, the gradients in x and y are
             sampled in a circle of radius 6s using wavelets of size 4s.
             We ensure the gradient wavelet size is even to ensure the
             wavelet pattern is balanced and symmetric around its center */
            int grad_wav_size = 2*cvRound( 2*s );
            //小波模板尺寸超出图像范围。把关键点设为无意义的点
            if( sum->rows < grad_wav_size || sum->cols < grad_wav_size )
            {
                /* when grad_wav_size is too big,
                 * the sampling of gradient will be meaningless
                 * mark keypoint for deletion. */
                kp.size = -1;
                continue;
            }

            float descriptor_dir = 360.f - 90.f;
            //考虑旋转
            if (upright == 0)
            {
                //生成新滤波器子块在原图像中的坐标点
                resizeHaarPattern( dx_s, dx_t, NX, 4, grad_wav_size, sum->cols );
                resizeHaarPattern( dy_s, dy_t, NY, 4, grad_wav_size, sum->cols );
                for( kk = 0, nangle = 0; kk < nOriSamples; kk++ )//nOriSamples = 169
                {
                    //定位相关扫描角度内的坐标点
                    int x = cvRound( center.x + apt[kk].x*s - (float)(grad_wav_size-1)/2 );
                    int y = cvRound( center.y + apt[kk].y*s - (float)(grad_wav_size-1)/2 );
                    if( y < 0 || y >= sum->rows - grad_wav_size ||
                        x < 0 || x >= sum->cols - grad_wav_size )
                        continue;
                    const int* ptr = &sum->at<int>(y, x);//获取原积分图像素值
                    float vx = calcHaarPattern( ptr, dx_t, 2 );//计算HarrX响应
                    float vy = calcHaarPattern( ptr, dy_t, 2 );//计算HarrY响应
                    //加权滤波
                    X[nangle] = vx*aptw[kk];
                    Y[nangle] = vy*aptw[kk];
                    nangle++;
                }//k
                if( nangle == 0 )
                {
                    // No gradient could be sampled because the keypoint is too
                    // near too one or more of the sides of the image. As we
                    // therefore cannot find a dominant direction, we skip this
                    // keypoint and mark it for later deletion from the sequence.
                    kp.size = -1;
                    continue;
                }//nangle == 0
                matX.cols = matY.cols = _angle.cols = nangle;
                cvCartToPolar( &matX, &matY, 0, &_angle, 1 );

                //求出主方向
                float bestx = 0, besty = 0, descriptor_mod = 0;
                for( i = 0; i < 360; i += SURF_ORI_SEARCH_INC )
                {
                    float sumx = 0, sumy = 0, temp_mod;
                    for( j = 0; j < nangle; j++ )
                    {
                        int d = std::abs(cvRound(angle[j]) - i);
                        if( d < ORI_WIN/2 || d > 360-ORI_WIN/2 )
                        {
                            sumx += X[j];
                            sumy += Y[j];
                        }//d
                    }//j
                    temp_mod = sumx*sumx + sumy*sumy;
                    if( temp_mod > descriptor_mod )
                    {
                        descriptor_mod = temp_mod;
                        bestx = sumx;
                        besty = sumy;
                    }//temp_mod > descriptor_mod
                }//i
                descriptor_dir = fastAtan2( -besty, bestx );
            }//upright == 0

            kp.angle = descriptor_dir;
            if( !descriptors || !descriptors->data )
                continue;

            /* Extract a window of pixels around the keypoint of size 20s */
            /* 用特征点周围20*s为边长的邻域 计算特征描写叙述子 */
            int win_size = (int)((PATCH_SZ+1)*s);//20*s邻域窗体大小
            CV_Assert( winbuf->cols >= win_size*win_size );
            Mat win(win_size, win_size, CV_8U, winbuf->data.ptr);

            if( !upright )//考虑旋转
            {
                descriptor_dir *= (float)(CV_PI/180);//转为弧度
                float sin_dir = -std::sin(descriptor_dir);
                float cos_dir =  std::cos(descriptor_dir);

                /* Subpixel interpolation version (slower). Subpixel not required since
                the pixels will all get averaged when we scale down to 20 pixels */
                /*
                float w[] = { cos_dir, sin_dir, center.x,
                -sin_dir, cos_dir , center.y };
                CvMat W = cvMat(2, 3, CV_32F, w);
                cvGetQuadrangleSubPix( img, &win, &W );
                */

                //旋转到主方向
                float win_offset = -(float)(win_size-1)/2;
                float start_x = center.x + win_offset*cos_dir + win_offset*sin_dir;
                float start_y = center.y - win_offset*sin_dir + win_offset*cos_dir;
                uchar* WIN = win.data;
//#if 0
//                // Nearest neighbour version (faster)
//                for( i = 0; i < win_size; i++, start_x += sin_dir, start_y += cos_dir )
//                {
//                    float pixel_x = start_x;
//                    float pixel_y = start_y;
//                    for( j = 0; j < win_size; j++, pixel_x += cos_dir, pixel_y -= sin_dir )
//                    {
//                        int x = std::min(std::max(cvRound(pixel_x), 0), img->cols-1);
//                        int y = std::min(std::max(cvRound(pixel_y), 0), img->rows-1);
//                        WIN[i*win_size + j] = img->at<uchar>(y, x);
//                    }
//                }
//#else
                int ncols1 = img->cols-1, nrows1 = img->rows-1;
                size_t imgstep = img->step;
                for( i = 0; i < win_size; i++, start_x += sin_dir, start_y += cos_dir )
                {
                    double pixel_x = start_x;
                    double pixel_y = start_y;
                    for( j = 0; j < win_size; j++, pixel_x += cos_dir, pixel_y -= sin_dir )
                    {
                        int ix = cvFloor(pixel_x), iy = cvFloor(pixel_y);
                        if( (unsigned)ix < (unsigned)ncols1 &&
                            (unsigned)iy < (unsigned)nrows1 )
                        {
                            float a = (float)(pixel_x - ix), b = (float)(pixel_y - iy);
                            const uchar* imgptr = &img->at<uchar>(iy, ix);
                            WIN[i*win_size + j] = (uchar)
                                cvRound(imgptr[0]*(1.f - a)*(1.f - b) +
                                        imgptr[1]*a*(1.f - b) +
                                        imgptr[imgstep]*(1.f - a)*b +
                                        imgptr[imgstep+1]*a*b);
                        }
                        else
                        {
                            int x = std::min(std::max(cvRound(pixel_x), 0), ncols1);
                            int y = std::min(std::max(cvRound(pixel_y), 0), nrows1);
                            WIN[i*win_size + j] = img->at<uchar>(y, x);
                        }
                    }//j
                }//i
#endif
            }//考虑旋转
            else//不考虑旋转
            {
                // extract rect - slightly optimized version of the code above
                // TODO: find faster code, as this is simply an extract rect operation,
                //       e.g. by using cvGetSubRect, problem is the border processing
                // descriptor_dir == 90 grad
                // sin_dir == 1
                // cos_dir == 0

                float win_offset = -(float)(win_size-1)/2;
                int start_x = cvRound(center.x + win_offset);
                int start_y = cvRound(center.y - win_offset);
                uchar* WIN = win.data;
                for( i = 0; i < win_size; i++, start_x++ )
                {
                    int pixel_x = start_x;
                    int pixel_y = start_y;
                    for( j = 0; j < win_size; j++, pixel_y-- )
                    {
                        int x = MAX( pixel_x, 0 );
                        int y = MAX( pixel_y, 0 );
                        x = MIN( x, img->cols-1 );
                        y = MIN( y, img->rows-1 );
                        WIN[i*win_size + j] = img->at<uchar>(y, x);
                    }//j
                }//i
            }//不考虑旋转
            // Scale the window to size PATCH_SZ so each pixel's size is s. This
            // makes calculating the gradients with wavelets of size 2s easy
            resize(win, _patch, _patch.size(), 0, 0, INTER_AREA);

            // Calculate gradients in x and y with wavelets of size 2s
            for( i = 0; i < PATCH_SZ; i++ )
                for( j = 0; j < PATCH_SZ; j++ )
                {
                    float dw = DW[i*PATCH_SZ + j];
                    float vx = (PATCH[i][j+1] - PATCH[i][j] + PATCH[i+1][j+1] - PATCH[i+1][j])*dw;
                    float vy = (PATCH[i+1][j] - PATCH[i][j] + PATCH[i+1][j+1] - PATCH[i][j+1])*dw;
                    DX[i][j] = vx;
                    DY[i][j] = vy;
                }

            // Construct the descriptor
            vec = descriptors->ptr<float>(k);
            for( kk = 0; kk < dsize; kk++ )
                vec[kk] = 0;
            double square_mag = 0;
            if( extended )// 128-bin descriptor
            {
                // 128-bin descriptor
                for( i = 0; i < 4; i++ )
                    for( j = 0; j < 4; j++ )
                    {
                        for(int y = i*5; y < i*5+5; y++ )
                        {
                            for(int x = j*5; x < j*5+5; x++ )
                            {
                                float tx = DX[y][x], ty = DY[y][x];
                                if( ty >= 0 )
                                {
                                    vec[0] += tx;
                                    vec[1] += (float)fabs(tx);
                                } else {
                                    vec[2] += tx;
                                    vec[3] += (float)fabs(tx);
                                }
                                if ( tx >= 0 )
                                {
                                    vec[4] += ty;
                                    vec[5] += (float)fabs(ty);
                                } else {
                                    vec[6] += ty;
                                    vec[7] += (float)fabs(ty);
                                }
                            }
                        }
                        for( kk = 0; kk < 8; kk++ )
                            square_mag += vec[kk]*vec[kk];
                        vec += 8;
                    }
            }
            else// 64-bin descriptor
            {
                // 64-bin descriptor
                for( i = 0; i < 4; i++ )
                    for( j = 0; j < 4; j++ )
                    {
                        for(int y = i*5; y < i*5+5; y++ )
                        {
                            for(int x = j*5; x < j*5+5; x++ )
                            {
                                float tx = DX[y][x], ty = DY[y][x];
                                vec[0] += tx; vec[1] += ty;
                                vec[2] += (float)fabs(tx); vec[3] += (float)fabs(ty);
                            }
                        }
                        for( kk = 0; kk < 4; kk++ )
                            square_mag += vec[kk]*vec[kk];
                        vec+=4;
                    }
            }

            // unit vector is essential for contrast invariance
            // 归一化描写叙述子以满足光照不变性
            vec = descriptors->ptr<float>(k);
            float scale = (float)(1./(sqrt(square_mag) + DBL_EPSILON));
            for( kk = 0; kk < dsize; kk++ )
                vec[kk] *= scale;
        }
    }

    // Parameters
    const Mat* img;
    const Mat* sum;
    vector<KeyPoint>* keypoints;
    Mat* descriptors;
    bool extended;
    bool upright;

    // Pre-calculated values
    int nOriSamples;
    vector<Point> apt;
    vector<float> aptw;
    vector<float> DW;
};