So I decided to build upon my ASL project that I did earlier this semester. Originally I was going to do a search engine using the ASL alphabet. But during the user testing session Anna had an idea to make it into an ASL learning game. What would happen is at the top of the screen images of the handshapes of letters would appear. The object is that you would copy the handshapes which would input a query (which you wouldn’t see) and then it would pull up the first google image of what you signed. Then you would type into an answer box what you think the word you signed meant. If you got it correct you’d move onto the next level, if not you’d have to guess again. When I started putting all of these ideas together I ran into many road blocks though. When I first started I was trying to use a LeapMotion but it wasn’t working with Processing at all. Then when I started with the flex sensors I couldn’t get the Arduino to work (turns out I had the wrong numbers put into the program, but whatever). Then I got the Arduino to work but I had to find all the various numbers for the letters again, and it wasn’t working as nicely as the last time I ran the ASL program. Next I just tried to get Processing to print the letter on the screen, but I couldn’t figure out how to get readings from the Arduino to be slower and I couldn’t figure out how to make the letters into one string. Because I couldn’t figure out this I wasn’t able to think of a way to get the letters into a query. So I don’t have the entire project all together… But I do have all the pieces of the code (except for the ASL Handshape pictures).
Here is the Processing code for getting the letters to show up on the screen.
import processing.serial.*;
Serial myPort;
int flexFore;
int flexNumbers;
void setup() {
size(640, 480);
// List all the available serial ports:
printArray(Serial.list());
// Print out the list and look for port your Microcontroller is on
// Finding the Arduino is not easy because they have weird names on the mac and on the pc they look like COM1.
// Ardnino is almost always the first one so you can usually use Serial.list()[0].
String portName = Serial.list()[5];
myPort = new Serial(this, portName, 9600);
myPort.readStringUntil('\n'); //clean out anything already in the buffer
size(640, 480);
}
void draw() {
background(255);
textSize(40);
fill(0);
if (flexNumbers >= 90 && flexNumbers <=100) {
if (flexFore >= 25 && flexFore < 40) {
text("A",width/2,height/2);
delay(1000);
}
}
if (flexNumbers >= 90&& flexNumbers <110) {
if (flexFore >=100 && flexFore <115) {
text("H",width/2,height/2);
delay(1000);
}
}
if (flexNumbers >=80 && flexNumbers <90) {
if (flexFore >= 100 && flexFore <115) {
text("G",width/2,height/2);
delay(1000);
}
}
if (flexNumbers >=60 && flexNumbers <70) {
if (flexFore >=50 && flexFore <65) {
text("C",width/2,height/2);
delay(1000);
}
}
if (flexNumbers >=50 && flexNumbers <60) {
if (flexFore >=40 && flexFore <55) {
text("F",width/2,height/2);
delay(1000);
}
}
if (flexNumbers >=70&& flexNumbers <78) {
if (flexFore >= 90 && flexFore <=115) {
text("D",width/2,height/2);
delay(1000);
}
}
else if (flexNumbers >=30 && flexNumbers <65) {
if (flexFore >=90 && flexFore <=115) {
text("B",width/2,height/2);
delay(1000);
}
}
if (flexNumbers <50) {
if (flexFore >= 0 && flexFore < 50) {
text("E",width/2,height/2);
delay(1000);
}
}
else if (flexNumbers > 110) {
if (flexFore >115 ) {
text(" ",width/2,height/2);
delay(1000);
}
}
}
void serialEvent(Serial _port) { //this is a callback function
if (myPort == null) return;
String input = myPort.readStringUntil('\n');
if (input != null) { //if a '\n' character has in fact now arrived
String[] parts = split(input, ","); //split the message into parts based on the comma
//the parts got put into a special kind of variable called an array that has slots
String var1 = parts[0]; //they came in the order they were sent
String var2 = parts[1]; //so the force will be in the first slot of the array fire in second
flexFore = int(var1); //Turn it into a number for eelipse in draw
flexNumbers = int(var2); //Turn it into a number for fill in draw
println(flexFore, flexNumbers);
}
}
And here is the query code:
import java.io.*;
String query = "";
void setup() {
size(640, 480);
background(0);
fill(255, 255, 255);
}
void draw() {
}
void queryGoogleImages(String _q) {
String url = "https://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + _q;
//this is a special url for getting google image results as json
println(url);
JSONObject result = loadJSONObject(url);
//make a network call and get the results back into a json object
println(result);
//make sure the results are not an error
int status = result.getInt("responseStatus");
if (status != 200) {
String reason = result.getString("responseDetails");
//this sucks that google does not allow you to do very many of these in a row
println("You are too fast, take a break." + reason);
return;
}
//go one level down in to the results to just get the data not the info about the package
JSONObject response = result.getJSONObject("responseData");
println(response);
//get the arrary of stuff you got back
JSONArray values = response.getJSONArray("results");
int vOffset = 20;
for (int i = 0; i < min(values.size(),3); i++) {
//pick out the picture information for each picture
JSONObject thisGuy = values.getJSONObject(i);
String r = thisGuy.getString("url");
int w = thisGuy.getInt("tbWidth");
int h = thisGuy.getInt("tbHeight");
println(r + " "+ w + " " + h );
PImage thisImage = loadImage(r);
if (thisImage != null) {
image(thisImage, 0, 0, 640, 460);
}
else {
text((i + 1) + " key", 10, vOffset+13);
rect(10, vOffset, w, h);
}
//make vOffset bigger so we will draw the next picture below this one.
vOffset += h;
}//end for
}
void keyPressed() {
if (keyCode == RETURN || keyCode == ENTER) {
background(0);
queryGoogleImages(query); //execute the query
query = ""; //clear the query
}
else if (key > 48 && key < 123) { //letters and numbers only
query = query + key;
text(query, 320, 470); //draw the query so far on the scrren
}
}
