I am trying to develop a scanner that can scan a page of a Passport with the camera.
So from a Passport page like this:
I'd like to crop out the marked part.
I have written code for edge detection using OpenCV which finds the contours and then approximates the largest quadrilateral. Finally it does a 4 point perspective transformation to get a top view of the image. The edge detection code look like this:
public static List<MatOfPoint> findContours(Mat src){
Mat img = src.clone();
src.release();
//find contours
double ratio = getScaleRatio(img.size());
int width = (int) (img.size().width / ratio);
int height = (int) (img.size().height / ratio);
Size newSize = new Size(width, height);
Mat resizedImg = new Mat(newSize, CvType.CV_8UC4);
Imgproc.resize(img, resizedImg, newSize);
Imgproc.medianBlur(resizedImg, resizedImg, 5);
Mat cannedImg = new Mat(newSize, CvType.CV_8UC1);
Imgproc.Canny(resizedImg, cannedImg, 70, 200, 3, true);
resizedImg.release();
Imgproc.threshold(cannedImg, cannedImg, 200, 255, Imgproc.THRESH_OTSU);
Mat dilatedImg = new Mat(newSize, CvType.CV_8UC1);
Mat morph = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
Imgproc.dilate(cannedImg, dilatedImg, morph, new Point(-1, -1), 2, 1, new Scalar(1));
cannedImg.release();
morph.release();
ArrayList<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();
Imgproc.findContours(dilatedImg, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
hierarchy.release();
Log.d(TAG, "contours found: " + contours.size());
Collections.sort(contours, new Comparator<MatOfPoint>() {
@Override
public int compare(MatOfPoint o1, MatOfPoint o2) {
return Double.valueOf(Imgproc.contourArea(o2)).compareTo(Imgproc.contourArea(o1));
}
});
return contours;
}
this code works for single page documents i.e ID cards, credit cards. Where the input is like this:
But doesn't work for passports as the top edge is not as distinctive, for example:
The inputs will be taken from camera preview on Android. Any idea how can I detect the passport page? I am using OpenCV 3.1. Thanks in Advance!
EDIT:
I am adding a few sample images.
1.
2.
3.
4.
EDIT 2:
I have tried drawing the top border myself so that a quadrilateral could be formed, but in that case the perspective transformation doesn't work as expected. Here's the complete code:
img = data.input.clone();
data.input.release();
//find contours
ratio = img.size().height/FIXED_HEIGHT;
width = (int) (img.size().width / ratio);
height = (int) (img.size().height / ratio);
newSize = new Size(width, height);
resizedImg = new Mat(newSize, CvType.CV_8UC4);
Imgproc.resize(img, resizedImg, newSize);
onNextStep(resizedImg);
Imgproc.medianBlur(resizedImg, resizedImg, 5);
onNextStep(resizedImg);
//Imgproc.line(resizedImg, new Point(0, 0), new Point(resizedImg.cols()-1, 0), new Scalar(0, 0, 0), 1);
cannedImg = new Mat(newSize, CvType.CV_8UC1);
Imgproc.Canny(resizedImg, cannedImg, 70, 200, 3, true);
resizedImg.release();
onNextStep(cannedImg);
Imgproc.threshold(cannedImg, cannedImg, 70, 255, Imgproc.THRESH_OTSU);
onNextStep(cannedImg);
dilatedImg = new Mat(newSize, CvType.CV_8UC1);
morph = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
Imgproc.dilate(cannedImg, dilatedImg, morph, new Point(-1, -1), 2, 1, new Scalar(1));
cannedImg.release();
morph.release();
onNextStep(dilatedImg);
contours = new ArrayList<>();
hierarchy = new Mat();
Imgproc.findContours(dilatedImg, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
hierarchy.release();
Log.d(TAG, "contours found: " + contours.size());
Collections.sort(contours, new Comparator<MatOfPoint>() {
@Override
public int compare(MatOfPoint o1, MatOfPoint o2) {
return Double.valueOf(Imgproc.contourArea(o2)).compareTo(Imgproc.contourArea(o1));
}
});
Imgproc.drawContours(dilatedImg, contours, 0, new Scalar(255, 255, 250));
onNextStep(dilatedImg);
//dilatedImg.release();
Rect box = Imgproc.boundingRect(contours.get(0));
Imgproc.line(dilatedImg, box.tl(), new Point(box.br().x, box.tl().y), new Scalar(255, 255, 255), 2);
onNextStep(dilatedImg);
contours = new ArrayList<>();
hierarchy = new Mat();
Imgproc.findContours(dilatedImg, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
hierarchy.release();
Log.d(TAG, "contours found: " + contours.size());
Collections.sort(contours, new Comparator<MatOfPoint>() {
@Override
public int compare(MatOfPoint o1, MatOfPoint o2) {
return Double.valueOf(Imgproc.contourArea(o2)).compareTo(Imgproc.contourArea(o1));
}
});
Imgproc.drawContours(dilatedImg, contours, 0, new Scalar(255, 255, 250));
onNextStep(dilatedImg);
dilatedImg.release();
rectContour = null;
foundPoints = null;
for(MatOfPoint contour:contours){
MatOfPoint2f mat = new MatOfPoint2f(contour.toArray());
double peri = Imgproc.arcLength(mat, true);
MatOfPoint2f approx = new MatOfPoint2f();
Imgproc.approxPolyDP(mat, approx, 0.02 * peri, true);
Point[] points = approx.toArray();
Log.d("SCANNER", "approx size " + points.length);
if (points.length == 4) {
Point[] spoints = CVProcessor.sortPoints(points);
if (CVProcessor.insideArea(spoints, newSize)) {
rectContour = contour;
foundPoints = spoints;
break;
}
}
}
if(rectContour != null){
Point[] scaledPoints = new Point[foundPoints.length];
for(int i = 0; i < foundPoints.length; i++){
scaledPoints[i] = new Point(foundPoints[i].x * ratio, foundPoints[i].y * ratio);
}
Log.d("SCANNER", "drawing lines");
Imgproc.line(img, scaledPoints[0], scaledPoints[1], new Scalar(250, 0, 0), 10);
Imgproc.line(img, scaledPoints[0], scaledPoints[3], new Scalar(250, 0, 0), 10);
Imgproc.line(img, scaledPoints[1], scaledPoints[2], new Scalar(250, 0, 0), 10);
Imgproc.line(img, scaledPoints[3], scaledPoints[2], new Scalar(250, 0, 0), 10);
}
