Revision history [back]

@Gilad Darmon since you want to detect the squares you can use some shape segmentation technique as it is described in the examples here and here. You can also combine these algorithms since they differ a bit in specific parts of the source code. Anyway, by just trying some modifications and some basic steps I got the following result:

I guess this is what you want. For the code check below. By the way I guess you are aware that your original image is distorted and as a consequence that affects the result. If you undistort your image most likely you will obtain better shapes on your detected squares. Plus there is this dirt on the window causing some problems and destroying the shape of one of the squares which as you can notice is not detected.

#include <iostream>
#include <opencv2/opencv.hpp>

using namespace std;
using namespace cv;

// helper function:
// finds a cosine of angle between vectors
// from pt0->pt1 and from pt0->pt2
static double angle( Point pt1, Point pt2, Point pt0 )
{
    double dx1 = pt1.x - pt0.x;
    double dy1 = pt1.y - pt0.y;
    double dx2 = pt2.x - pt0.x;
    double dy2 = pt2.y - pt0.y;
    return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
}

// returns sequence of squares detected on the image.
// the sequence is stored in the specified memory storage
static void findSquares( const Mat& image, vector<vector<Point> >& squares )
{
    squares.clear();

    Mat pyr, timg, gray0(image.size(), CV_8U), gray;

    // down-scale and upscale the image to filter out the noise
    pyrDown(image, pyr, Size(image.cols/2, image.rows/2));
    pyrUp(pyr, timg, image.size());
    vector<vector<Point> > contours;

    // find squares in every color plane of the image
    for( int c = 0; c < 3; c++ )
    {
        int ch[] = {c, 0};
        mixChannels(&timg, 1, &gray0, 1, ch, 1);

        // try several threshold levels
        int N = 5;
        for( int l = 0; l < N; l++ )
        {
            // hack: use Canny instead of zero threshold level.
            // Canny helps to catch squares with gradient shading
            if( l == 0 )
            {
                // apply Canny. Take the upper threshold from slider
                // and set the lower to 0 (which forces edges merging)
                Canny(gray0, gray, 0, 50, 5);
                // dilate canny output to remove potential
                // holes between edge segments
                dilate(gray, gray, Mat(), Point(-1,-1));
            }
            else
            {
                // apply threshold if l!=0:
                //     tgray(x,y) = gray(x,y) < (l+1)*255/N ? 255 : 0
                gray = gray0 >= (l+1)*255/N;
            }

            // find contours and store them all as a list
            findContours(gray, contours, RETR_LIST, CHAIN_APPROX_SIMPLE);

            vector<Point> approx;

            // test each contour
            for( size_t i = 0; i < contours.size(); i++ )
            {
                // approximate contour with accuracy proportional
                // to the contour perimeter
                approxPolyDP(Mat(contours[i]), approx, arcLength(Mat(contours[i]), true)*0.02, true);

                // square contours should have 4 vertices after approximation
                // relatively large area (to filter out noisy contours)
                // and be convex.
                // Note: absolute value of an area is used because
                // area may be positive or negative - in accordance with the
                // contour orientation
                if( approx.size() == 4 &&
                    fabs(contourArea(Mat(approx))) > 1000 &&
                    isContourConvex(Mat(approx)) )
                {
                    double maxCosine = 0;

                    for( int j = 2; j < 5; j++ )
                    {
                        // find the maximum cosine of the angle between joint edges
                        double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
                        maxCosine = MAX(maxCosine, cosine);
                    }

                    // detect ratio that corresponds to a rectangle or a square
                    cv::Rect r = cv::boundingRect(contours[i]);
                    double ratio = std::abs(1 - (double)r.width / r.height);
                    // if cosines of all angles are small
                    // (all angles are ~90 degree)
                    //  and the ratio does not corresponds to
                    // a rectangle then write quandrange
                    // vertices to resultant sequence
                    if( maxCosine < 0.3 && ratio <= 0.09)
                        squares.push_back(approx);
                }
            }
        }
    }
}

// the function draws all the squares in the image
static void drawSquares( Mat& image, const vector<vector<Point> >& squares )
{
    for( size_t i = 0; i < squares.size(); i++ )
    {
        const Point* p = &squares[i][0];
        int n = (int)squares[i].size();
        polylines(image, &p, &n, 1, true, Scalar(0,255,0), 1, LINE_AA);
    }

    imshow("squares", image);
}

int main()
{
    // load image
    Mat src = imread("squares.png");

    // check that it is loaded correctly
    if(!src.data || src.empty())
        cerr << "Problem loading image!!!" << endl;

    imshow("src", src);

    // transform to grayscale
    Mat gray;
    cvtColor(src, gray, CV_BGR2GRAY);

    // get the binary version
    Mat bin;
    adaptiveThreshold(~gray, bin, 255, CV_ADAPTIVE_THRESH_MEAN_C, THRESH_BINARY, 29, -1);
//    imshow("bin", bin);

    // remove some noise
    medianBlur(bin, bin, 7);
//    imshow("median", ~bin);

    // find squares and draw them
    Mat tmp;
    cvtColor(~bin, tmp, CV_GRAY2BGR);
    vector<vector<Point> > squares;
    findSquares(tmp, squares);
    drawSquares(src, squares);

    waitKey();
    return 0;
}