Revision history [back]

I'm using a goodFeaturesToTrack and calcOpticalFlowPyrLK to stabilize video but the result has a black border, how to remove it.

sample result:

https://gfycat.com/unawarecomposedgoldfinch

my stabilization code :

   Mat VideoStab::stabilize(Mat frame_1, Mat frame_2)
{
    cvtColor(frame_1, frame1, COLOR_BGR2GRAY);
    cvtColor(frame_2, frame2, COLOR_BGR2GRAY);

    int vert_border = HORIZONTAL_BORDER_CROP * frame_1.rows / frame_1.cols;

    vector <Point2f> features1, features2;
    vector <Point2f> goodFeatures1, goodFeatures2;
    vector <uchar> status;
    vector <float> err;

    //Estimating the features in frame1 and frame2
    goodFeaturesToTrack(frame1, features1, 200, 0.01  , 30 );
    calcOpticalFlowPyrLK(frame1, frame2, features1, features2, status, err );

    for(size_t i=0; i < status.size(); i++)
    {
        if(status[i])
        {
            goodFeatures1.push_back(features1[i]);
            goodFeatures2.push_back(features2[i]);
        }
    }

    try {

    //All the parameters scale, angle, and translation are stored in affine
    affine = estimateRigidTransform(goodFeatures1, goodFeatures2, true);

    cout<<affine;
    //flush(cout);

   // affine = affineTransform(goodFeatures1 , goodFeatures2);
    if (affine.empty()) {
        return frame_2;
    }
    dx = affine.at<double>(0,2);
    dy = affine.at<double>(1,2);
    da = atan2(affine.at<double>(1,0), affine.at<double>(0,0));
    ds_x = affine.at<double>(0,0)/cos(da);
    ds_y = affine.at<double>(1,1)/cos(da);

    sx = ds_x;
    sy = ds_y;

    sum_transX += dx;
    sum_transY += dy;
    sum_thetha += da;
    sum_scaleX += ds_x;
    sum_scaleY += ds_y;


    //Don't calculate the predicted state of Kalman Filter on 1st iteration
    if(k==1)
    {
        k++;
    }
    else
    {
        Kalman_Filter(&scaleX , &scaleY , &thetha , &transX , &transY);

    }

    diff_scaleX = scaleX - sum_scaleX;
    diff_scaleY = scaleY - sum_scaleY;
    diff_transX = transX - sum_transX;
    diff_transY = transY - sum_transY;
    diff_thetha = thetha - sum_thetha;

    ds_x = ds_x + diff_scaleX;
    ds_y = ds_y + diff_scaleY;
    dx = dx + diff_transX;
    dy = dy + diff_transY;
    da = da + diff_thetha;

    //Creating the smoothed parameters matrix
    smoothedMat.at<double>(0,0) = sx * cos(da);
    smoothedMat.at<double>(0,1) = sx * -sin(da);
    smoothedMat.at<double>(1,0) = sy * sin(da);
    smoothedMat.at<double>(1,1) = sy * cos(da);

    smoothedMat.at<double>(0,2) = dx;
    smoothedMat.at<double>(1,2) = dy;

    //Uncomment if you want to see smoothed values
    //cout<<smoothedMat;
    //flush(cout);

    //Warp the new frame using the smoothed parameters
    warpAffine(frame_1, smoothedFrame, smoothedMat, frame_2.size());

    //Crop the smoothed frame a little to eliminate black region due to Kalman Filter
    smoothedFrame = smoothedFrame(Range(vert_border, smoothedFrame.rows-vert_border), Range(HORIZONTAL_BORDER_CROP, smoothedFrame.cols-HORIZONTAL_BORDER_CROP));

    resize(smoothedFrame, smoothedFrame, frame_2.size());

    //Change the value of test if you want to see both unstabilized and stabilized video
    if(test)
    {
        Mat canvas = Mat::zeros(frame_2.rows, frame_2.cols*2+10, frame_2.type());

        frame_1.copyTo(canvas(Range::all(), Range(0, smoothedFrame.cols)));

        smoothedFrame.copyTo(canvas(Range::all(), Range(smoothedFrame.cols+10, smoothedFrame.cols*2+10)));

        if(canvas.cols > 1920)
        {
            resize(canvas, canvas, Size(canvas.cols/2, canvas.rows/2));
        }
        imshow("before and after", canvas);
    }

    return smoothedFrame;

    } catch(Exception e){

        return frame_2;
    }

}