Your Favorite Pixel

Now we want to dive into the image and find parts of it. Pulling one thing out of a scene is very easy for your brain and very hard for your software.
It is pretty easy to write software to look for something close to a certain color or brightness. Even for this your expectations might exceed your coding because your eye unconsciously adjusts to changes in ambient lighting where your code won't. Look for things of a certain shape, pattern, texture or image are yet even harder.
In many applications it is possible to contrive the scene to make the things you are interested in look for very distinctive. For instance wear an orange glove or bright light on your nose.
Here is a loop for looking through all the pixels to find the brightest one.

	          public void draw() {
			if (video.available()) {
				video.read();
				float brightestSoFar = 0; //start out very far away
				int winnerX = 0;
				int winnerY = 0;
				for(int row = 0; row < video.height; row++){
					for(int col = 0; col < video.width; col++){
						int offset = row*width + col;
						int thisPixel = video.pixels[offset];
						float brightness = brightness(thisPixel);
						//check to see if this beats the world record for brightness
						if ( brightness > brightestSoFar ){
							winnerX = col;
							winnerY = row;
							brightestSoFar = brightness; //up the world record to this brightness
						}
					}
				}
				image(video,0,0);
				fill(255,0,0);
				ellipse(winnerX, winnerY,30,30); //draw a circle over the winner
			}
		}

Favorite Among Favorites

The previous example will find a bright spot but might move around quite a lot within a bright area.
Rather than look for a single pixel, you probably want all the surrounding pixels that are bright. For this you ask each pixel if they are close enough, that is within a threshold of brightness. To start with we will just find the average position of all the qualifying pixels.

           public void draw() {
		if (video.available()) {
			video.read();
			int threshold = 20;
			for(int row = 0; row < video.height; row++){
				for(int col = 0; col < video.width; col++){
					int offset = row*width + col;
					int thisPixel = video.pixels[offset];
					float brightness = brightness(thisPixel);
                                        //check to see if it is over a threshold of brightness
					if (brightness < threshold) {
                                                video.pixels[offset] = 0;  //to debug the qualifying pixesl
						totalQualified++;
						sumX = sumX + col;
						sumY = sumY + row;
					}
				}
			}
			image(video,0,0);
			int aveX = sumX/totalQualified;
			int aveY = sumY/totalQualified;
			ellipse(aveX-5,aveY-5,10,10);
		}

}

Adjusting the Threshold

Instead of looking for one winner you are comparing to a threshold value but how do you work out that threshold. If you are in very controlled lighting situations you can just do it by trial and error and hard code it. In most circumstances it is better to be able to adjust it manually or determine it dynamically based on the average brightness of the whole scene:

                 public void keyPressed() {
				if (key == 's') {
					video.settings();
				} else if (key == '-') {
					threshold--;
					println("Threshold " + threshold);
				} else if (key == '=') {
					threshold++;
					println("Threshold " + threshold);
				} else if (key == 'a'){
					video.read();
					video.loadPixels();
                                        float totalBrightness = 0;
					for(int i = 0; i < video.pixels.length; i++){
						totalBrightness += brightness(video.pixels[i]);
					}
					threshold = 2* totalBrightness/video.pixels.length; 
                                        //now pixels have to be twice as bright as average to qualify.	
				}
			}

Put A Box Around an Area

Finding the center point (average) of all the many qualifying points gets rid of the jumpiness of the previous example. But we are still getting just a single point.
To get the area of the object you are looking for you can keep track of a bounding box around all the qualifying pixels. Keeping track of the area can give you an idea of how close something is to the camera so in effect you can derive z (depth) in addition to the row and column. It give you a rudimentary start in identifying things by shape. You will need at least four numbers to keep track of the four corners around the qualifying pixels (or you can use the Rectangle Object in Java).

	     public void draw() {
		if (video.available()) {
			video.read();

			int leftMost = width;  //starts out as the right most
			int topMost = height; // starts out as the bottom most
			int rightMost = 0; // starts out left most
			int bottomMost = 0; //sarts out top most

			for (int row = 0; row < video.height; row++) {
				for (int col = 0; col < video.width; col++) {
					int offset = row * width + col;

					int thisPixel = video.pixels[offset];
					float h = hue(thisPixel);
					float s = saturation(thisPixel);
					float b = brightness(thisPixel);
					float closeness = dist(h, s, b, goalHue, goalSaturation, goalBrightness);
					if (closeness < threshold) {
						video.pixels[offset] = 0; //color it black for debuggin
						if (col < leftMost) leftMost = col;
						if (col > rightMost) rightMost = col;
						if (row > bottomMost ) bottomMost = row;
						if (row < topMost) topMost = row;
					}
				}
			}
			image(video, 0, 0);
			stroke(255, 0, 255);
			fill(0, 255, 0, 0);
			rect(leftMost, topMost, rightMost-leftMost, bottomMost-topMost);
		}

}

Find Multiple Areas

The next step might be to track multiple areas in the scene. You need to keep track of the leftMost, rightMost, topMost, bottomMost for every potential area in the scene. It might make for a lot of variables. Instead we can just use the Java Rectangle class to wrap these variables up into a single object and give functions like contains, add, grow thrown in.
Now we need a variable that can hold multiple things. Hopefully you are familiar with array variables. An regular array might work to store all the Rectangles you find in the scene. But then you would have to know how many you are going to find ahead of time. Java has something called an ArrayList that can hold multiple things like an array but can resize dynamically as you need to add and subtract. Here is a comparison between ArrayLists ordinary arrays.

Rectangle[] myArray = new Rectangle[10]; //stuck with a size of 10 ArrayList myList = new ArrayList(); //start with an empty list and add them as needed

myArray[0] = new Rectangle(10,10,50,50); /add something to the list myList.add(newRectangle(10,10,50,50);

myArray.length; //learn the length myList.size(); //

thisRect = myArray[0]; //pull something out of the array thisRect = (Rectangle) myList.get(0); //notice you have to "cast" it as a rectangle on the way out

Another difference between regular arrays and ArrayLists is that you don't have to specify the type of stuff you are storing in an ArrayList but you do in an Array. This is why when we went to retrieve things from the list you have have tell them what type they are. You do this by casting them like in a play for instance puting "(Rectangle)" to the left of something to cast it as a Rectangle.
There is a new way of using ArrayLists that have types build into them. This will allow eclipse to find type mismatches in your code before you run it. Syntactically you create an ArrayList of a particular type by wedging in <Type> in a couple of places. Then the indignity of always having to do all that casting is lifted.

ArrayList<Rectangle> myList = new ArrayList<Rectangle>(); myRect = myList.get(0); //look no casting!

import java.awt.Rectangle; import java.util.ArrayList;

import blobs.Blob;

import processing.core.PApplet; import processing.video.Capture;

public class TrackRects extends PApplet {

	Capture video;// regular processing libary

	float goalRed = 200;

	float goalGreen = 0;

	float goalBlue = 0;

	int threshold = 40;

	int reach = 5;

	long elapsedTime;

	static public void main(String _args[]) {
		PApplet.main(new String[] { "tracking.TrackRects" });
	}

	public void setup() {
		size(640, 480);
		video = new Capture(this, width, height);
	}

	public void draw() {
		if (video.available()) {


			video.read();
			long startTime = millis();
			//video.filter(BLUR, 2);

			elapsedTime = millis() - startTime;
			//println(elapsedTime);  
			//uncomment this line or press 't' to see how long the blur takes.

			ArrayList<Rectangle> boxes = new ArrayList<Rectangle>();
			for (int row = 0; row < video.height; row++) {
				for (int col = 0; col < video.width; col++) {
					int offset = row * width + col;
					int thisPixel = video.pixels[offset];
					float r = red(thisPixel);
					float g = green(thisPixel);
					float b = blue(thisPixel);
					float closeness = dist(r, g, b, goalRed, goalGreen, goalBlue);
					if (closeness < threshold) {
						video.pixels[offset] = 0;
						//be pessimistic
						boolean foundAHome = false;
						//look throught the existing
						for (int i = 0; i < boxes.size(); i++) {
							Rectangle existingBox =   boxes.get(i);
							//is this spot in an  existing box
							Rectangle inflatedBox = new Rectangle(existingBox); //copy the existing box
							inflatedBox.grow(reach, reach); //widen it's reach
							if (inflatedBox.contains(col, row)) {
								existingBox.add(col,row);
								foundAHome = true; //no need to make a new one
								break; //no need to look through the rest of the boxes
							}
						}
						//if this does not belong to one of the existing boxes make a new one at this place
						if (foundAHome == false) boxes.add(new Rectangle(col, row,0,0));
					}
				}
			}

			//consolidate(boxes,0,0);
			image(video, 0, 0);
			fill(0, 0, 0, 0);
			stroke(255, 0, 0);
			for (int i = 0; i < boxes.size(); i++) {
				Rectangle thisBox =  boxes.get(i);
				rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
			}


		}
	}



	public void mousePressed() {

		int thisPixel = video.pixels[mouseY * width + mouseX];
		goalRed = red(thisPixel);
		goalGreen = green(thisPixel);
		goalBlue = blue(thisPixel);
		println("Goal" + goalRed + " " + goalGreen + " " + goalBlue);
	}

	public void keyPressed() {
		if (key == 's') {
			video.settings();
		} else if (key == '-') {
			threshold--;
			println("Threshold " + threshold);
		} else if (key == '=') {
			threshold++;
			println("Threshold " + threshold);
		} else if (key == '_') {
			reach--;
			println("reach " + reach);
		} else if (key == '+') {
			reach++;
			println("reach " + reach);
		} else if (key == 't') {

			println("Elapsedtime " + elapsedTime);

		}
	}

}