Tag Archives: ASL

ASL Learning Game (attempt)

So I decided to build upon my ASL project that I did earlier this semester. Originally I was going to do a search engine using the ASL alphabet. But during the user testing session Anna had an idea to make it into an ASL learning game. What would happen is at the top of the screen images of the handshapes of letters would appear. The object is that you would copy the handshapes which would input a query (which you wouldn’t see) and then it would pull up the first google image of what you signed. Then you would type into an answer box what you think the word you signed meant. If you got it correct you’d move onto the next level, if not you’d have to guess again. When I started putting all of these ideas together I ran into many road blocks though. When I first started I was trying to use a LeapMotion but it wasn’t working with Processing at all. Then when I started with the flex sensors I couldn’t get the Arduino to work (turns out I had the wrong numbers put into the program, but whatever). Then I got the Arduino to work but I had to find all the various numbers for the letters again, and it wasn’t working as nicely as the last time I ran the ASL program. Next I just tried to get Processing to print the letter on the screen, but I couldn’t figure out how to get readings from the Arduino to be slower and I couldn’t figure out how to make the letters into one string. Because I couldn’t figure out this I wasn’t able to think of a way to get the letters into a query. So I don’t have the entire project all together… But I do have all the pieces of the code (except for the ASL Handshape pictures).

Here is the Processing code for getting the letters to show up on the screen.

import processing.serial.*;

Serial myPort;
int flexFore;
int flexNumbers;

void setup() {
  size(640, 480);
  // List all the available serial ports:
  printArray(Serial.list());
  // Print out the list and look for port your Microcontroller is on
  // Finding the Arduino is not easy because they have weird names on the mac and on the pc they look like COM1.
  // Ardnino is almost always the first one so you can usually use Serial.list()[0].
  String portName = Serial.list()[5];
  myPort = new Serial(this, portName, 9600);
  myPort.readStringUntil('\n');  //clean out anything already in the buffer
  size(640, 480);
}

void draw() {
  background(255);
  textSize(40);
  fill(0);
  if (flexNumbers >= 90 && flexNumbers <=100) {
    if (flexFore >= 25 && flexFore < 40) {
      text("A",width/2,height/2);
      delay(1000);
    }
  }

  if (flexNumbers >= 90&& flexNumbers <110) {
    if (flexFore >=100 && flexFore <115) {
      text("H",width/2,height/2);
      delay(1000);
    }
  }
  if (flexNumbers >=80 && flexNumbers <90) {
    if (flexFore >= 100 && flexFore <115) {
      text("G",width/2,height/2);
      delay(1000);
    }
  }
   if (flexNumbers >=60 && flexNumbers <70) {
    if (flexFore >=50 && flexFore <65) {
text("C",width/2,height/2);
      delay(1000);
    }
  }
  if (flexNumbers >=50 && flexNumbers <60) {
    if (flexFore >=40 && flexFore <55) {
text("F",width/2,height/2);
      delay(1000);
    }
  }
  if (flexNumbers >=70&& flexNumbers <78) {
    if (flexFore >= 90 && flexFore <=115) {
text("D",width/2,height/2);
      delay(1000);
    }
  }
  else if (flexNumbers >=30 && flexNumbers <65) {
    if (flexFore >=90 && flexFore <=115) {
text("B",width/2,height/2);
      delay(1000);
    }
  }
  if (flexNumbers <50) {
    if (flexFore >= 0 && flexFore < 50) {
text("E",width/2,height/2);
      delay(1000);
    }
  }
  else if (flexNumbers > 110) {
    if (flexFore >115 ) {
text(" ",width/2,height/2);
      delay(1000);
    }
  }
}

void serialEvent(Serial _port) { //this is a callback function
  if (myPort == null) return;
  String input = myPort.readStringUntil('\n');
  if (input != null) {  //if a '\n' character has in fact now arrived
    String[] parts = split(input, ",");  //split the message into parts based on the comma
    //the parts got put into a special kind of variable called an array that has slots
    String var1 = parts[0]; //they came in the order they were sent 
    String var2 = parts[1]; //so the force will be in the first slot of the array fire in second
    flexFore = int(var1);  //Turn it into a number for eelipse in draw
    flexNumbers = int(var2);  //Turn it into a number for fill in draw
    println(flexFore, flexNumbers);
  }
}

And here is the query code:

import java.io.*;
String query = "";

void setup() {
  size(640, 480);
  background(0);
  fill(255, 255, 255);
}

void draw() {
}

void queryGoogleImages(String _q) {
  String url = "https://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + _q;
  //this is a special url for getting google image results as json
  println(url);
  JSONObject result =  loadJSONObject(url);
  //make a network call and get the results back into a json object
  println(result);
  //make sure the results are not an error
  int status = result.getInt("responseStatus");
  if (status != 200) {
    String reason = result.getString("responseDetails");
    //this sucks that google does not allow you to do very many of these in a row
    println("You are too fast, take a break." + reason);
    return;
  }
  //go one level down in to the results to just get the data not the info about the package
  JSONObject response = result.getJSONObject("responseData");
  println(response);
  //get the arrary of stuff you got back
  JSONArray values = response.getJSONArray("results");
  int vOffset = 20;
  for (int i = 0; i < min(values.size(),3); i++) {
    //pick out the picture information for each picture
    JSONObject thisGuy = values.getJSONObject(i); 
    String r = thisGuy.getString("url");
    int w = thisGuy.getInt("tbWidth");
    int h = thisGuy.getInt("tbHeight");
    println(r + " "+ w + " " + h );
    PImage thisImage = loadImage(r);
    if (thisImage != null) {
      image(thisImage, 0, 0, 640, 460);
    }
    else {
      text((i + 1) + " key", 10, vOffset+13);
      rect(10, vOffset, w, h);
    }
    //make vOffset bigger so we will draw the next picture below this one.
    vOffset  += h;

  }//end for
}

void keyPressed() {
  if (keyCode == RETURN || keyCode == ENTER) {
    background(0);
    queryGoogleImages(query); //execute the query
    query = "";  //clear the query
  } 
  else if (key > 48 && key < 123) {  //letters and numbers only
    query = query + key;
    text(query, 320, 470);  //draw the query so far on the scrren
  }
}

 

ASL to LED (AKA American Sign LED-uage)

For my project I wanted to combine American Sign Language and what we’ve been working on in class. So I decided to translate the sign language alphabet into Morse code (done through flashes on an LED). I used two flex sensors, one on my thumb and one on my forefinger to do so. For this project I only did the letters A through H because to go past that I would need another flex sensor as well as an accelerometer to accurately transmit the letters. However, for A through H only the thumb sensor would technically be needed, but for accuracy (and I bought the thing so I figured why not) I used the one on the forefinger as well. It took me a while to figure out exactly how to code it and I feel like there would be a way to shorten down the code, but I did not know how at this point. But the biggest time consumer was figuring out the sensor ranges needed to code. And here is the video explaining some of this and a demonstration.

//all the alphabet used to monitor my accuracy of handshapes
// to make sure it was picking up the right letter.
int A = 1;
int B = 2;
int C = 3;
int D = 4;
int E = 5;
int F = 6;
int G = 7;
int H = 8;
int dot = 500;
int dash = 1000;
int pause = 200;
void setup(){
  pinMode (8, OUTPUT);
  Serial.begin(9600);

}

void loop(){
  int input = analogRead(A1);
  int inputF = analogRead (A2);
  int flexFore = map (inputF, 320, 557, 0, 100);
  int flexNumbers = map(input, 360, 540, 0, 100);

  if (flexNumbers >= 90 && flexNumbers <=100){
    if (flexFore >= 25 && flexFore < 35){
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
      Serial.println(A);
    delay(3000);
    }
  }

   else if (flexNumbers >= 80 && flexNumbers <90){
     if (flexFore >=80 && flexFore <95) {
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
       Serial.println(H);
    delay(3000);
     }
  }
  else if (flexNumbers >=70 && flexNumbers <80) {
    if (flexFore >= 80 && flexFore <95){
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);  
    Serial.println(G);
    delay(3000);
    }
  }
  else if (flexNumbers >=60 && flexNumbers <70){
    if (flexFore >=50 && flexFore <60){
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);  
    Serial.println(C);
    delay(3000);
    }
  }
  else if (flexNumbers >=50 && flexNumbers <60) {
    if (flexFore >=30 && flexFore <40){
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
      Serial.println(F);
    delay(3000);
    }
  }
  else if (flexNumbers >=40 && flexNumbers <50) {
    if (flexFore >= 90 && flexFore <=100){
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
      Serial.println(D);
    delay(3000);
    }
  }
  else if (flexNumbers >=30 && flexNumbers <40){
    if (flexFore >=90 && flexFore <=100){
    digitalWrite (8, HIGH);
    delay (dash);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
    delay (pause);
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
      Serial.println(B);
    delay(3000);
    }
  }
  else if (flexNumbers <30){
    if (flexFore >= 0 && flexFore < 20){
    digitalWrite (8, HIGH);
    delay (dot);
    digitalWrite (8, LOW);
      Serial.println(E);
    delay(3000);
    }
  }

}