Contour perspective warp

edit

You need to find first the Homography. Have a look at this tutorial. Then, it would be easier to reverse the usage of scene points and object points to get the transformation that gives you the rectangle in the way you want (instead of finding the orientation of the rectangle in the scene, you try to find the fronto-parallel orientation, if that does make sense).

You could also have a look at the affine transformation estimation made by the estimageRigidTransform function or the findTransformEEC function.

May be something like this

int main(int argc, char* argv[])
{
    VideoCapture v(0);
    Mat im=imread("14596150505143125.jpg");
    Mat img,dst;
    double thresh=0;
    vector<vector <Point>> ctr;
    cvtColor(im,img,CV_BGR2GRAY);
    threshold(img,dst,thresh,255,THRESH_OTSU);
    findContours(dst,ctr,CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);

    int idx,maxCtr=-1;
    double pi = acos(-1.0);
    for (int i = 0; i<ctr.size();i++)
        if (saturate_cast<int>(ctr[i].size())>maxCtr)
        {
            maxCtr=ctr[i].size();
            idx=i;
        }
    RotatedRect r=minAreaRect(ctr[idx]);
    cout<<r.center<<"\t Size ="<<r.size<<"\tAngle="<<r.angle<<"\n";
    Mat rt1=(Mat_<float>(3,3)<< 1,0,-r.center.x,0,1,-r.center.y,0,0,1);
    double theta=(270-r.angle)/180*pi;
    Mat rr=(Mat_<float>(3,3)<< cos(theta),-sin(theta),0,sin(theta),cos(theta),0,0,0,1);
    Mat rt2=(Mat_<float>(3,3)<< 1,0,r.center.x,0,1,r.center.y,0,0,1);
    Mat rAffine;
    Mat rt=(rt2*rr*rt1);
    rt(Rect(0,0,3,2)).copyTo(rAffine);
    Mat imR;
    cout<<rAffine;
    warpAffine(im, imR,rAffine, im.size());
    imshow("result",imR);
    waitKey();
    return 0;
}

If there is a perpesctive you can use this program. You have to check if point in ctrApprox and rotatedrect are in same order

#include "opencv2/imgproc/imgproc.hpp"
#include "opencv2/videoio/videoio.hpp"
#include "opencv2/calib3d.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/highgui.hpp"

#include <iostream>
#include <fstream>
#include <algorithm>
#include <ctype.h>

using namespace cv;
using namespace std;



int main(int argc, char* argv[])
{
    VideoCapture v(0);
    Mat im=imread("C:/Users/Laurent.PC-LAURENT-VISI/Downloads/14596150505143125.jpg");
    Mat img,dst;
    double thresh=0;
    vector<vector <Point>> ctr;
    cvtColor(im,img,CV_BGR2GRAY);
    threshold(img,dst,thresh,255,THRESH_OTSU);
    findContours(dst,ctr,CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);

    int idx,maxCtr=-1;
    double pi = acos(-1.0);
    for (int i = 0; i<ctr.size();i++)
        if (saturate_cast<int>(ctr[i].size())>maxCtr)
        {
            maxCtr=ctr[i].size();
            idx=i;
        }
    vector<Point> ctrApprox;
    double epsilon=2;
    do
    {
        approxPolyDP(ctr[idx], ctrApprox, epsilon, true);
        epsilon++;    
    }
    while (ctrApprox.size()>4);
    RotatedRect r=minAreaRect(ctr[idx]);
    vector <Point2f> p2(4);
    p2[0] = Point2f(r.center.x - r.size.width/2,r.center.x - r.size.height/2);
    p2[1] = Point2f(r.center.x - r.size.width/2,r.center.x + r.size.height/2);
    p2[2] = Point2f(r.center.x + r.size.width/2,r.center.x + r.size.height/2);
    p2[3] = Point2f(r.center.x + r.size.width/2,r.center.x - r.size.height/2);
    Mat rth=findHomography(ctrApprox,p2);
    warpPerspective(im, imR,rth, Size(im.size().width*2,im.size().height*2));
    imshow("result",imR);
    waitKey();
    return 0;
}