For my ICM final project, I decided to continue work on my previous project “Thought Bubbles” from the midterm. My goal was to record people’s thoughts, allow the user to blow bubbles, and give the illusion that the bubbles actually contained the users voice, so that when they popped his or her voice would come out of the bubble.
For video of the project in action (at the ITP 2010 Winter Showcase), check here and here
Coding help for this project came from a variety of sources:
Daniel Shiffman and his book Learning Processing for various examples on building functions and other basic coding help
Tom Igoe for Arduino and Processing Examples for Serial Communication
Minim Sound Library for Processing including examples on Audio Snippets & Audio Recorder objects
OpenCV for Processing including examples on Blob Tracking
Here is my source code for Processing.
import processing.video.*;
import hypermedia.video.*;
import ddf.minim.*;
import processing.serial.*;
//Serial to Arduino
Serial myPort; //Serial Port talking to the Arduino
int motorControl; //int to tell motors what to do
//Piecees needed for sound playing
Minim minim;
AudioInput input; //input from microphone
AudioRecorder think; //recording the thoughts
//playing back that sound
AudioSnippet thought;
float val; //used to pan audio
float vol; //used to gain audio
int sample;
String idea;
float level; //check if the user is speaking or blowing
float levelThreshold; //threshold for speaking versus blowing
int levelAdjust;
//Pieces needed for bubble tracking
Capture ps3; //using processing.video to capture image
OpenCV vid; //OpenCV video object
Blob[] bubbles; //ArrayList keeping track of the bubbles
int bubbleSmall; //min blob size
int bubbleLarge; //max blob size
int increm; //increment the string to save to differen files
String[] numPeople; //an array to be saved to the text file with the value of increm
boolean recording = false; //Is recording active
boolean newPerson = false; //Is there a new recording
boolean playing = false; //Is a sound playing
String bubblething;
//audioPlaying is a counter to determine when again I can play a sound
int audioPlaying = 0;
boolean audioCounter = false;
//duration of recorded files and used to check if the sound is playing
int soundCounter = 100;
int counter = 0; //used to calculate when to end the recording
String savior; //string name of the .wav file to be saved
int maxBubble = 100;
int bubbleRange = 50;
int[] totalbubbles = new int[20]; //an array int list to keep track of total bubbles on the screen
int[] xLoc = new int[maxBubble];
int[] pxLoc = new int[maxBubble];
int[] yLoc = new int[maxBubble];
int[] pyLoc = new int[maxBubble];
boolean popped;
void setup() {
println("start");
println(Capture.list());
size(340, 240);
numPeople = loadStrings("numPeople.txt");
println(numPeople);
increm = int(numPeople[0]);
minim = new Minim(this);
input = minim.getLineIn(Minim.STEREO); //set the input
myPort = new Serial (this, Serial.list()[0], 9600);
ps3 = new Capture(this, width, height, "Sony HD Eye for PS3 (SLEH 00201)", 1000);
vid = new OpenCV(this);
vid.capture(width,height);
idea = "person" + (increm-1) + ".wav";
thought = minim.loadSnippet("person0.wav");
levelThreshold = .05;
//smooth();
bubbleSmall = 30;
bubbleLarge = 500;
}
void draw() {
if (audioCounter == true) {
if (millis() - audioPlaying > soundCounter * 6) {
audioCounter = false;
playing = false;
}
}
level = input.mix.level();
println(level);
//levelThreshold = .03 * levelAdjust + .1;
if ((recording)) {
fill(0);
text("Recording", width/2, height/2);
counter++;
if (counter > soundCounter) {
recordingReset();
}
}
else {
if (level > levelThreshold) {
bubbleMachine();
}
else {
motorControl = 'b';
//zprintln("no bubbles");
}
// // map the mouse position to the range of the pan
// val = map(mouseX, 0, width, -1, 1);
// // if a pan control is not available, this will do nothing
// thought.setPan(val);
// vol = map(mouseY, 0, height, -10, 0);
// // if a pan control is not available this will report zero
// fill(255, 0, 0);
// thought.setGain(vol);
// text("The current pan is " + thought.getPan() + ".", 5, 20);
// text("The current volume is " + thought.getGain() + ".", 5, 30);
fill(0);
//write the current requested motorState to the arduino
myPort.write(motorControl);
if (ps3.available()) {
ps3.read();
vid.copy(ps3);
image(vid.image(), 0, 0);
vid.threshold(80);
bubbles = vid.blobs(bubbleSmall, bubbleLarge, maxBubble, false);
for (int i = 0; i < bubbles.length; i++) { fill(0, 26, 87); ellipse(bubbles[i].centroid.x, bubbles[i].centroid.y, 20, 20); //bubbles[i].length/5, bubbles[i].length/5); if (bubbles[i].centroid.x > 0) { //saving BubbleLocations
xLoc[i] = bubbles[i].centroid.x;
}
if (bubbles[i].centroid.y > 0) {
yLoc[i] = bubbles[i].centroid.y;
}
if (bubbles.length > 0) {
for (int j = 0; j < bubbles.length; j++) { } if ((xLoc[1] - pxLoc[1]) > bubbleRange || (yLoc[1] - pyLoc[1]) > bubbleRange) {
//if (i>1){
fill(255, 0, 0);
text("Bubble Popped!", width - 50, 20);
playSound();
//}
}
}
fill(0);
bubblething = "Bubble " + i;
text(bubblething, bubbles[i].centroid.x + 10, bubbles[i].centroid.y + 10);
pxLoc[i] = xLoc[i];
//bubbleswerehere.println("frame end");
}
stroke(1);
fill(0);
line(0, height/3, width, height/3);
line(0, 2 * height/3, width, 2 * height/3);
line(width/3, 0, width/3, height);
line(2 * width/3, 0, 2 * width/3, height);
vid.threshold(80);
}
}
// text("The current pan is " + thought.getPan() + ".", 5, 20);
//text("The current gain is " + thought.getGain() + ".", 5, 30);
}
void serialEvent (Serial myPort) {
int inByte = myPort.read();
if ((inByte > 47) && (58 > inByte)) {
levelAdjust = inByte - 48;
//levelThreshold = levelAdjust * .005 + .02;
//println(levelAdjust);
}
if (inByte == 'N') {
myPort.write('b');
motorControl = 'b';
savior = "person" + increm + ".wav"; //creating the string
think = minim.createRecorder(input, savior, false);
think.beginRecord();
counter = 0;
recording = true;
}
}
void playSound() {
if (playing == false) {
playing = true;
audioCounter = true;
println("play Sound test");
audioPlaying = millis();
if (newPerson) {
idea = "person" + (increm - 1) + ".wav";
thought = minim.loadSnippet(idea);
thought.setGain(60);
thought.play();
sample = floor(random(0, increm));
idea = "person" + sample + ".wav";
println(idea);
thought = minim.loadSnippet(idea);
newPerson = false;
}
else {
println("Play Sample " + thought);
thought.setGain(60);
thought.play();
sample = floor(random(0, increm));
idea = "person" + sample + ".wav";
thought = minim.loadSnippet(idea);
}
}
else {
println("Playing Already");
}
}
void bubbleMachine() {
motorControl = 'a';
println("blowing");
}
void recordingReset() {
think.endRecord(); //stop recording
think.save(); //save the recording
recording = false;
increm++;
numPeople[0] = str(increm);
saveStrings("numPeople.txt", numPeople );
println("saved");
newPerson = true;
myPort.write('o');
}
void keyPressed() {
if (key == 'z') {
levelThreshold = levelThreshold + .01;
println (levelThreshold);
}
else if (key == 'x') {
levelThreshold = levelThreshold - .01;
println (levelThreshold);
}
}
void stop() {
input.close();
minim.stop();
vid.stop();
super.stop();
}