Interactive Display using OpenCV and Sonic rangefinder sensor
Design/Development Stage
We have incorporated the OpenCV library into our code, and have managed to get the interactive display to respond to our body movements. Instead of using FACE tracking, we found the BLOB tracking to be more responsive and more dependable. With Face Tracking, unless your face is always pointed at the camera, it will not be recognized. If your face is turned to the side, the face recognition will not work.
The first step was to create an environment where we can understand how the OpenCV library works with Processing, and lay the foundation for adding our interactive layers in the future. We started with some code posted by Parag K. Mital in Feb 2009 for Digital Media Studio Project 2009, University of Edinburgh. The code was written for the Eclipse Java environment, but it was easily converted to Processing code.
We stripped down the original version of the code in order to make it as responsive as possible. The plan was to take a snapshot of the background and perform a background subtraction – this way we can detect when something new was introduced in the camera’s field of vision, and track it.
On the right, you can see the background ”memory” image. The left side shows that it detected an object (my face) and is tracking it. The “Centroid” is the center coordinate on the x axis which we need to track in order to determine if the object is moving right or left.
// Written by Parag K. Mital // 16 Feb 2009 //
import hypermedia.video.*; // Open CV library
import java.awt.*; // java awt library
OpenCV cv = null;
// camera width/height
int capture_width = 320;
int capture_height = 240;
// threshold for background subtraction
int threshold = 125;
// some variables to control the contrast/brightness of the camera image
int contrast = 0, brightness = 0;
// for drawing text
PFont font;
// these boolean values will be used to trigger parts of the code - we
// can use the keyPressed method to set these boolean values with the keyboard
boolean draw_blobs=true, draw_centroid=true, show_difference=true;
// we are going to track the centroid of all blobs and keep the previous
// and current estimate
int previous_centroid_x = capture_width / 2;
int previous_centroid_y = capture_height / 2;
int current_centroid_x = capture_width / 2;
int current_centroid_y = capture_height / 2;
void setup() {
// Size of the window
size(capture_width*2, capture_height*3);
// open video stream
cv = new OpenCV( this );
//cv.capture( 640, 480 );
// Setup our capture device using opencv
cv.capture(capture_width, capture_height, 0);
// Setup font to use the Andale Mono type font (this file is in the data folder)
font = loadFont("AndaleMono.vlw");
textFont(font);
}
void draw() {
// set the background color to black
background(0);
// display the frame per second of our program. if this gets too low,
// our program will appear less interactive as the latency will be higher
text("fps: " + frameRate, 10, capture_height+10);
cv.read(); // grab frame from camera
//cv.threshold(80); // set black & white threshold
// adjust the Contrast/Brightness (stored in OpenCV.BUFFER)
cv.contrast(contrast);
cv.brightness(brightness);
// call the method for background subtraction
doBackgroundSubtraction();
// and blob detection
doBlobDetection();
}
void doBackgroundSubtraction() {
// OpenCV has 3 images you can access,
// the SOURCE (original image from camera or movie)
// the BUFFER (image after any operations like convert, brightness, threshold, etc...)
// the MEMORY (the image stored when OpenCV.remember(..) is called)
//image( cv.image(OpenCV.SOURCE), 0, 0 );
//text( "original source image", 10, 10 );
// use the OpenCV function convert to convert
// the source image to grayscale and store in the
// OpenCV.BUFFER
//cv.convert(OpenCV.GRAY);
//image( cv.image(OpenCV.BUFFER), capture_width, 0 );
//text( "grayscale buffer image", capture_width+10, 10);
// Calculate the absolute difference between the
// image in the OpenCV Memory and the current image
cv.absDiff();
// Create a binary matrix in the OpenCV.BUFFER that
// has values >= threshold set to 1,
// and values < threshold set to 0
cv.threshold(threshold);
text( "threshold: " + threshold, capture_width+10, capture_height+10 );
// Blur the image in the OpenCV.BUFFER
//cv.blur(OpenCV.GAUSSIAN, 11);
if (show_difference)
{
image( cv.image(OpenCV.BUFFER), 0, 0 );
text( "absoulte-difference image", 10, 10 );
}
image( cv.image(OpenCV.MEMORY), capture_width, 0 );
text( "memory image", capture_width+10, 10 );
}
void doBlobDetection() {
// Do the blob detection
Blob[] blobs = cv.blobs(100, capture_width*capture_height/3, 1, false);
/* public Blob[] blobs(int minArea,
int maxArea,
int maxBlobs,
boolean findHoles) */
// Pushing a matrix allows any transformations such as rotate, translate, or scale,
// to stay within the same matrix. once we "popMatrix()", then all the commands for that matrix
// are gone for any subsequent drawing.
pushMatrix();
// since we translate after we pushed a matrix, every drawing command afterwards will be affected
// up until we popMatrix(). then this translate will have no effect on drawing commands after
// the popMatrix().
translate(capture_width, 0);
// We are going to keep track of the total x,y centroid locations to find
// an average centroid location of all blobs
int total_x = 0;
int total_y = 0;
// we loop through all of the blobs found by cv.blobs(..)
for ( int blob_num = 0; blob_num < blobs.length; blob_num++ ) {
if (draw_blobs)
{
// get the bounding box from the blob detection and draw it
Rectangle bounding_box = blobs[blob_num].rectangle;
noFill();
stroke(128);
this.rect( bounding_box.x, bounding_box.y, bounding_box.width, bounding_box.height );
}
// accumulate the centroids
Point centroid = blobs[blob_num].centroid;
total_x += centroid.x;
total_y += centroid.y;
text("Centeroid: " + (total_x * 1), -capture_width+10, capture_height+20);
}
if (blobs.length > 0)
{
// by keeping the previous centroid, we can do an interpolation between
// the current centroid and the previous one. this will make any tracking results
// seem smoother. a better way would be to use a low pass filter over a vector
// of centroid locations. i.e. keep more than just 1 previous location and
// find a better way of interpolating the current centroid from them.
previous_centroid_x = current_centroid_x;
previous_centroid_y = current_centroid_y;
current_centroid_x = (total_x/blobs.length + previous_centroid_x) / 2;
current_centroid_y = (total_y/blobs.length + previous_centroid_x) / 2;
}
if (draw_centroid)
{
// draw a crosshair at the centroid location
this.ellipse(current_centroid_x, current_centroid_y, 5, 5);
this.line( current_centroid_x-5, current_centroid_y, current_centroid_x+5, current_centroid_y );
this.line( current_centroid_x, current_centroid_y-5, current_centroid_x, current_centroid_y+5 );
}
popMatrix();
}
// keyPressed is a processing function that lets us know when a user
// has pressed a key. the variable, key, will be set to a character of the keyboard.
void keyPressed() {
if ( key == ' ' ) {
//cv.flip(cv.FLIP_HORIZONTAL);
cv.remember(OpenCV.SOURCE);
}
if ( key == '+') {
if (threshold >= 255) threshold = 255;
else threshold++;
}
if ( key == '-' ) {
if (threshold <= 0) threshold = 0;
else threshold--;
}
if (key == 'b' ) {
draw_blobs = !draw_blobs;
}
if (key == 'c' ) {
draw_centroid = !draw_centroid;
}
if (key == 'd') {
show_difference = !show_difference;
}
}