This is the algorithm I'm trying to implement:
- Assume sideways camera movement. (Like looking out of the window on a train.)
- Take a photo
- Wait until 50% (configurable) of what was on the photo has moved out of the frame.
- Repeat step 2-3
I need OpenCV for step 3. My best guess so far is this:
- Analyse the photo with
goodFeaturesToTrackto get 100 features. - Take a new photo.
- Use
calcOpticalFlowPyrLKto find the new location of the features. - Use the status output from
calcOpticalFlowPyrLKto see how many of the features was found on the latest photo. - If less than 50%, go back to step 2.
Alternative step 4-5:
- Check the average sideways movement of detected features.
- If more than half of image width, go back to step 2.
I expected prevPts and nextPts from calcOpticalFlowPyrLK two be two lists of the same features, with their locations on two images. So 7th item on both lists represents the 7th feature from the first image. In prevPts it's the location on the first image, in nextPts it's the location on the latest image.
But often they are not. They can be completely different features, so detecting movement by comparing the locations is completely unreliable.
Is there a better way? Or maybe I'm using goodFeaturesToTrack the wrong way?
Here's my complete code. It's Java for now, I'm going to use it on Android later:
import static org.opencv.highgui.Highgui.CV_CAP_PROP_FRAME_HEIGHT;
import static org.opencv.highgui.Highgui.CV_CAP_PROP_FRAME_WIDTH;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import javax.imageio.ImageIO;
import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.core.MatOfFloat;
import org.opencv.core.MatOfPoint;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.highgui.Highgui;
import org.opencv.highgui.VideoCapture;
import org.opencv.imgproc.Imgproc;
import org.opencv.video.Video;
public class Camera {
private static final int MAX_CORNERS = 100;
private static final double QUALITY_LEVEL = 0.05;
private static final double MIN_DISTANCE = 0.01;
private static final int MAX_LEVEL = 2;
private final Size winSize = new Size(15, 15);
private JFrame jframe = new JFrame();
private JLabel label;
private VideoCapture cap;
private Mat frame;
private Mat frame_gray;
private Mat old;
private Mat old_gray;
private boolean first;
public static void main(String[] args) {
Camera cam = new Camera();
while (true) {
cam.detectMove(0.5);
}
}
public Camera() {
first = true;
System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
cap = new VideoCapture(0);
if (!cap.isOpened()) {
throw new RuntimeException("Could not capture video.");
}
int dWidth = (int) cap.get(CV_CAP_PROP_FRAME_WIDTH);
int dHeight = (int) cap.get(CV_CAP_PROP_FRAME_HEIGHT);
System.out.println("Frame size : " + dWidth + " x " + dHeight);
old = new Mat(640, 480, CvType.CV_8UC4);
old_gray = new Mat(640, 480, CvType.CV_8UC1);
frame = new Mat(640, 480, CvType.CV_8UC4);
frame_gray = new Mat(640, 480, CvType.CV_8UC1);
}
public void detectMove(double d) {
MatOfPoint initial = new MatOfPoint();
MatOfPoint2f prevPts = new MatOfPoint2f();
if (first) {
first = false;
cap.read(old);
Imgproc.cvtColor(old, old_gray, Imgproc.COLOR_BGR2GRAY);
}
Imgproc.goodFeaturesToTrack(old_gray, initial, MAX_CORNERS,
QUALITY_LEVEL, MIN_DISTANCE);
initial.convertTo(prevPts, CvType.CV_32FC2);
Point[] prevPtsArray = prevPts.toArray();
int features = prevPtsArray.length;
boolean goalDetected = false;
int active_corners = 0;
while (!goalDetected) {
cap.read(frame);
Imgproc.cvtColor(frame, frame_gray, Imgproc.COLOR_BGR2GRAY);
MatOfPoint2f nextPts = new MatOfPoint2f();
MatOfByte status = new MatOfByte();
MatOfFloat err = new MatOfFloat();
Video.calcOpticalFlowPyrLK(old_gray, frame_gray, prevPts, nextPts,
status, err, winSize, MAX_LEVEL);
byte[] statusArray = status.toArray();
Point[] nextPtsArray = nextPts.toArray();
// System.out.println(nextPtsArray.length);
// System.out.println("Points: " + initial.size());
active_corners = 0;
double sideways_translation = 0;
for (int i = 0; i < nextPtsArray.length; i++) {
if (statusArray[i] == 0)
continue;
sideways_translation += nextPtsArray[i].x - prevPtsArray[i].x;
active_corners++;
// System.out.println(statusArray[i]);
Core.circle(frame, prevPtsArray[i], 3, new Scalar(0, 0, 255));
Core.circle(frame, nextPtsArray[i], 1, new Scalar(255, 255, 0));
Core.line(frame, prevPtsArray[i], nextPtsArray[i], new Scalar(
255, 255, 0), 1);
}
sideways_translation /= active_corners;
String text = active_corners+"";
Core.putText(frame, text, new Point(10, 30),
Core.FONT_HERSHEY_SIMPLEX, 1, new Scalar(0, 0, 255));
Core.putText(frame, ""+sideways_translation, new Point(10, 60),
Core.FONT_HERSHEY_SIMPLEX, 1, new Scalar(0, 0, 255));
showResult(frame);
if (active_corners < features*d) {
goalDetected = true;
}
}
System.out.println("Detected! Only " + active_corners + " features found, needed " + ((int)(features*d)));
frame.copyTo(old);
frame_gray.copyTo(old_gray);
}
public void showResult(Mat img) {
Imgproc.resize(img, img, new Size(640, 480));
MatOfByte matOfByte = new MatOfByte();
Highgui.imencode(".jpg", img, matOfByte);
byte[] byteArray = matOfByte.toArray();
BufferedImage bufImage = null;
try {
InputStream in = new ByteArrayInputStream(byteArray);
bufImage = ImageIO.read(in);
if (label == null) {
label = new JLabel(new ImageIcon(bufImage));
jframe.getContentPane().add(label);
jframe.pack();
jframe.setVisible(true);
} else {
label.setIcon(new ImageIcon(bufImage));
}
} catch (Exception e) {
e.printStackTrace();
}
}
}