Painting with Computer Vision

photo 2

For my final project, I knew that I wanted to incorporate everything we had learned for both the Arduino and Processing. I knew that I waned it to be playful, tactile and interactive. After a few sketches and a few suggestions from my lovely peers, I came up with a glove that can be used to digitally draw on a computer screen or projection. The glove works by sending out an infrared signal from the tip of the pointer finger, which the camera on the computer reads and converts to a colorful pen. The size of the pen is regulated by a flex sensor that runs along the bottom of the same finger. The color of the pen is regulated using a potentiometer. The code lends itself to easy changes of the pen colors, but as I have set them now, they run through red, orange, yellow, green, blue and purple, in that order. Pressing any key will clear the drawing surface. Also, to give credit where credit is due, I have to thank Imogen Heap for her inspiring musical gloves (and Jeremiah for sharing the video) and Golan Levin’s brightness tracking code, which helped me to work out a bunch of kinks and led me to abandon the color tracking that I used in the earlier versions of this glove. The song I use is The Fifth Hand by Parra for Cuva.

photo 1

Here is a video of the glove in action, I apologize for the slight lag, the pen gets a little jumpy when quicktime is also recording:

Click Me!!!

And finally, here is my code:

import ddf.minim.spi.*;
import ddf.minim.signals.*;
import ddf.minim.*;
import ddf.minim.analysis.*;
import ddf.minim.ugens.*;
import ddf.minim.effects.*;

float flip;
float penSize; 
int penColor;
import processing.serial.*;

Capture cam;
Serial flexPort;

Minim minim;
AudioPlayer sou;

void setup() {
  size(640, 480); 
  cam = new Capture(this, width, height, 30);
  background (0);
  String portName = Serial.list()[7];
  flexPort = new Serial(this, portName, 9600);
  minim = new Minim(this);
  sou = minim.loadFile("The Fifth Hand.mp3");

void draw() {
  if (cam.available()) {;
    int brightestX = 0; 
    int brightestY = 0; 
    float brightestValue = 0; 
    int index = 0;
    for (int y = 0; y < cam.height; y++) {
      for (int x = 0; x < cam.width; x++) {
        // Get the color stored in the pixel
        int pixelValue = cam.pixels[index];
        // Determine the brightness of the pixel
        float pixelBrightness = brightness(pixelValue);
        if (pixelBrightness > brightestValue) {
          brightestValue = pixelBrightness;
          brightestY = y;
          brightestX = x;
      //mirror screen
      flip = (cam.width - brightestX);
     if ((penColor >= 0) && (penColor <=170)) {
      fill (#DC143C); //Red
    else if ((penColor >= 171) && (penColor <=341)) {
      fill (#FF8C00); //Orange
    else if ((penColor >= 342) && (penColor <=512)) {
      fill (#FFFF66); //Yellow
    else if ((penColor >= 513) && (penColor <=683)) {
      fill (#90EE90); //Green
    else if ((penColor >= 684) && (penColor <=853)) {
      fill (#4682B4); //Blue
    else if ((penColor >= 854) && (penColor <=1023)) {
      fill (#9400D3); //Purple

    ellipse(flip, brightestY, 10+(penSize-10), 10+(penSize-10));

void serialEvent(Serial _port) { 
  if (flexPort == null) return;
  String input = flexPort.readStringUntil('\n');
  if (input != null) {  
    String[] parts = split(input, ","); 
    String var1 = parts[0]; 
    String var2 = parts[1]; 
    penSize = int(var1);  
    penColor = int(var2);  


void keyPressed(){
 background (0);


and Arduino side:

void setup(){
  pinMode (A0, INPUT);
  pinMode (4, OUTPUT);
  pinMode (A1, INPUT);


void loop(){
  digitalWrite (4, HIGH);
  int flex = analogRead(A0);
  // Serial.println (flex);
  int trueFlex = map (flex, 30, 200, 100, 0);
  int twist = analogRead (A1);
  Serial.print (trueFlex); 
  Serial.print (twist); 


Leave a Reply