Revision history [back]

Here is an answer to this: The idea is to check whether every pixel is in the region respect to rectangle edges:

for(int i=0; i<myImage.rows; ++i)
{
    for(int j=0; j<myImage.cols; ++j)
    {
        if(isInRotatedRect( Point(j,i), rotated_rect))
        {
            myImage.at<uchar>(i,j) = 0;
        }
    }
}

for computing isInRotatedRect:

bool isInRotatedRect(Point p, RotatedRect rot)
{
    double line_pro[4];
    Point2f vertices[4];
    rRect.points(vertices);
    for(int i=0; i<4; ++i)
    {
        line_pro[i] = computeProduct(p, vertices[i], vertices[(i+1)%4]);
    }
    if(line_pro[1]*line_pro[3]<0 && line_pro[0]*line_pro[2]<0)
    {
        return true;
    }
    return false;
}

//finding whether  point p is at the left or right side of rectangle edge ab.  
double computeProduct(Point p, Point2f a, Point2f b)
{
    double m = (a.y-b.y) / (a.x-b.x);
    double c = a.y - k * a.x;
    return m * p.x - p.y + c;
}

A better and more intuitive way would be rather than doing this for every image make a zero one mask and the cross product original image! This may be more efficient specially in video.

Here is an answer to this: The idea is to check whether every pixel is in the region respect to rectangle edges:

for(int i=0; i<myImage.rows; ++i)
{
    for(int j=0; j<myImage.cols; ++j)
    {
        if(isInRotatedRect( Point(j,i), rotated_rect))
        {
            myImage.at<uchar>(i,j) = 0;
        }
    }
}

for computing isInRotatedRect:

bool isInRotatedRect(Point p, RotatedRect rot)
rRect)
{
    double line_pro[4];
    Point2f vertices[4];
    rRect.points(vertices);
    for(int i=0; i<4; ++i)
    {
        line_pro[i] = computeProduct(p, vertices[i], vertices[(i+1)%4]);
    }
    if(line_pro[1]*line_pro[3]<0 && line_pro[0]*line_pro[2]<0)
    {
        return true;
    }
    return false;
}

//finding whether  point p is at the left or right side of rectangle edge ab.  
double computeProduct(Point p, Point2f a, Point2f b)
{
    double m = (a.y-b.y) / (a.x-b.x);
    double c = a.y - k * a.x;
    return m * p.x - p.y + c;
}

A better and more intuitive way would be rather than doing this for every image make a zero one mask and the cross product original image! This may be more efficient specially in video.