Face the Web

Allows people to congregate in a layer on top of your web browser. Using a webcam attached to your computer the application grabs your face and sends it to other people looking at the same web page. Your face is displayed on other peoples’ browser at the place on the page you are most interested in as expressed by your cursor location. The web page becomes a substrate for gathering. In the future I would like to degrade the resolution of faces of people interested in things further away on the page and change to peer to peer architecture for the actual transmission of the images. This was done as the Thursday project in the 7 projects in 7 days (5 in 5 for me, … er make that 4 in 5) festival at ITP.

Tech: OpenCV processing libraries for the face detection but I would probably change that. I used IP cameras for the video capture. It also used a new capability in Java 1.6 for overlaying on on other applications. Unfortunately it only allows for a global translucency for the frame, not pixel by pixel transparency. It does allow for not rectilinear shapes so I might change it to that. Below is the code:

Client:

import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UTFDataFormatException;
import java.net.Socket;
import java.net.UnknownHostException;

import javax.swing.*;

import com.sun.image.codec.jpeg.ImageFormatException;
import com.sun.image.codec.jpeg.JPEGCodec;
import com.sun.image.codec.jpeg.JPEGEncodeParam;
import com.sun.image.codec.jpeg.JPEGImageDecoder;
import com.sun.image.codec.jpeg.JPEGImageEncoder;
import com.sun.image.codec.jpeg.TruncatedFileException;

import pFaceDetect.PFaceDetect;
import processing.core.PApplet;
import processing.core.PImage;
import processing.net.Server;
import vxp.AxisPixelSource;
import vxp.CaptureAxisCamera;

public class FaceTheWeb extends PApplet {
Server myServer;

int endOfMessageChar = 10;

Socket mySocket;

int socketPort = 9001;

String ip = “localhost”;

AxisPixelSource video;

boolean reply = true;

static String[] args;

int otherx, othery;

PImage other;

int myConnectionNumber = -1;

PFaceDetect face;

// PImage img;
public void setup() {
size(screen.width,screen.height);
setLocation(0,0);
JFrame.setDefaultLookAndFeelDecorated(true);
com.sun.awt.AWTUtilities.setWindowOpacity(this.frame, 0.5f);
video = new AxisPixelSource(“128.122.151.189”, 320, 240, false);

face = new PFaceDetect(this, video.getVideoWidth(), video.getVideoHeight(), “haarcascade_frontalface_default.xml”);

try {
mySocket = new Socket(ip, socketPort);
} catch (UnknownHostException e) {
println(“Could not find host address”);
} catch (IOException e) {
println(“Could not connect to host”);
}
println(“Connecting as client at ” + socketPort + ” on ” + ip);
new TeleListener(mySocket);

}

static public void main(String _args[]) {
PApplet.main(new String[] { “FaceTheWeb” });

}

public void draw() {
background(255,255,255,0);

if (other != null) {
image(other, otherx, othery, other.width, other.height);
}

}

public void sendPicture() {
if (mySocket != null && video != null) {
BufferedImage img = video.getImage();

face.findFaces(new PImage(img));
int[][] res = face.getFaces();
if (res.length > 0) {
// println(“Sendit”);
int x = res[0][0];
int y = res[0][1];
int w = res[0][2];
int h = res[0][3];
// rect(x, y, w, h);
BufferedImage bi = video.getImage().getSubimage(x, y, w, h);
new Sender(mySocket, bi);
/*
* for (int i = res.length – 1; i > res.length – 2; i–) { int x = res[i][0]; int y = res[i][1]; int w = res[i][2]; int h = res[i][3]; rect(x, y, w, h); BufferedImage bi = video.getImage().getSubimage(x, x, w, h); new Sender(mySocket, bi); }
*/
}
}

}

public void mousePressed() {
sendPicture();
}

public void incoming(PImage _img, int _connectionNumber) {
// println(“INcoming”);
if (_img != null) other = _img;

}

public class TeleListener extends Thread {

DataInputStream dis;

DataOutputStream dout;

int connectionNumber = 0;

TeleListener(Socket _c) {

try {
dis = new DataInputStream(_c.getInputStream());
dout = new DataOutputStream(_c.getOutputStream());
} catch (IOException e) {
println(“Couldn’t Connect”);
}

start();
}

public void run() {

while (true) {
int size = 0;
int connectionNumber = 0;

try {
String header = null;
try {

header = dis.readUTF();
} catch (UTFDataFormatException e) {
System.out.println(” UTFDataFormatException exeption “);
continue;
}
if (header == null) break;
try {
String[] messageParts = header.split(“,”);
size = Integer.parseInt(messageParts[1]);
otherx = Integer.parseInt(messageParts[2]);
othery = Integer.parseInt(messageParts[3]);

} catch (RuntimeException e) {
System.out.println(” RuntimeException exeption “);
continue;
}

if (size == 0) continue;

// make a buffer that size to accept the incoming bytes
byte[] incoming = new byte[size];
dis.readFully(incoming);
PImage myImage = null;
myImage = byteArrayToPImage(incoming);

incoming(myImage, connectionNumber);
// println(connectionNumber + ” Client got” + size);
} catch (NumberFormatException e) {
System.out.println(“Text to number problem.”);
break;
} catch (IOException e) {
System.out.println(“Nothing came in.”);
break;
}

}
}
}

public class Sender extends Thread {
DataOutputStream dos;

DataInputStream dis;

BufferedImage img;

Sender(Socket _c, BufferedImage _image) {
try {
dos = new DataOutputStream(_c.getOutputStream());
dis = new DataInputStream(_c.getInputStream());
} catch (IOException e) {
println(“Couldn’t Connect”);
}
img = _image;
start();
}

public void run() {

byte[] imageInBytes = imageToByteArray(img);
// tell them how many bytes to expect
String outSize = String.valueOf(imageInBytes.length);
String header = “enclosure,” + outSize + “,” + mouseX + “,” + mouseY;

try {
DataOutputStream dos = new DataOutputStream(mySocket.getOutputStream());
dos.writeUTF(header);
dos.flush();
dos.write(imageInBytes);
dos.flush();
} catch (IOException e) {
e.printStackTrace();
}

}

}

public byte[] imageToByteArray(BufferedImage _image) {

// bit if a hack to convert PImage to a Buffered Image, draw it on the buffered image
// ((BufferedImage) image).getRGB(x, y, w, h, img.pixels, offset, img.width);

// BufferedImage bi = new BufferedImage(_image.width, _image.height, BufferedImage.TYPE_INT_RGB);

// bi.setRGB(0, 0, _image.width, _image.height, _image.pixels, 0, _image.width);
// compress and turn into a byte array
// okay just to copy and paste this method

ByteArrayOutputStream baOut = new ByteArrayOutputStream();
JPEGImageEncoder encoder = JPEGCodec.createJPEGEncoder(baOut);
JPEGEncodeParam param = encoder.getDefaultJPEGEncodeParam(_image);
param.setQuality(0.9f, false);
encoder.setJPEGEncodeParam(param);
try {
encoder.encode(_image);
baOut.flush();
} catch (ImageFormatException e) {
System.out.println(“could not form image” + e);
} catch (IOException e) {
System.out.println(“could not encode image” + e);
}

return baOut.toByteArray();
}

public PImage byteArrayToPImage(byte[] _pic) {
// decompress into a buffferedImage
// okay just to copy and paste this method
ByteArrayInputStream in = new ByteArrayInputStream((byte[]) _pic);
JPEGImageDecoder decoder = JPEGCodec.createJPEGDecoder(in);
BufferedImage snapShot = null;
try {
snapShot = decoder.decodeAsBufferedImage();
} catch (ImageFormatException e) {
System.out.println(“Could not make array into a jpeg. ” + _pic.length);

} catch (TruncatedFileException e) {
System.out.println(“Truncated File” + _pic.length);
} catch (IOException e) {
System.out.println(“IO Making file” + _pic.length);
}

if (snapShot == null) return null;
return new PImage(snapShot);
}

}

Server:

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.ArrayList;

/*
* Created on Feb 7, 2005
*
*/

/**
* @author Dan O’Sullivan
*
* Allows conections to come in and spawns thread to take care of them
*/
public class FaceToWebServer {
int lastConnectionNumber = 0;

ArrayList allConnections = new ArrayList();

public static void main(String[] args) {
new FaceToWebServer();
}

FaceToWebServer() {
ServerSocket frontDoor;
int portNum = 9001;
try {
frontDoor = new ServerSocket(portNum);
System.out.println(“Set up a door at ” + InetAddress.getLocalHost() + ” ” + frontDoor.getLocalPort());
while (true) {
System.out.println(“Waiting for a new connection”);
Socket newSocket = frontDoor.accept();// sits here and waits for // someone to knock on the door
System.out.println(newSocket.getRemoteSocketAddress() + ” knocked on the door and I let them in”);
// farm it out immediately to another thread
ServerThread newServerThread = new ServerThread(newSocket);
allConnections.add(newServerThread); // add it to the list of connections
newServerThread.setConnectionID(lastConnectionNumber);
lastConnectionNumber++;
newServerThread.start();// this calls the run method in the thread
}
} catch (IOException e) {
System.out.println(“Had a problem making a front door” + e);
}
}

synchronized public void tellEveryone(ServerThread _me, String _header, byte[] _enclosure) {

for (int i = 0; i < allConnections.size(); i++) {

ServerThread thisConnection = (ServerThread) allConnections.get(i);
if (thisConnection == _me) continue; //don’t send back to yourself
//insert uniqueName into the header
String outHeader = _header+ “,” + _me.getConnectionID() + “,” + thisConnection.getConnectionID() ;
System.out.println(“Server Distributes ” + outHeader);
new ServerSenderThread(thisConnection.dout,outHeader,_enclosure);

}
}

synchronized public void removeMe(ServerThread _which) {
allConnections.remove(_which);

}

/**
* @author Dan O’Sullivan A thread that listens text and relays it to other people Try to add and introduction that asks for their name and then preppends name to all comments Try adding functionality where you can wisper to a specific person
*/
public class ServerThread extends Thread {
Socket mySocket;

DataInputStream din;

public DataOutputStream dout;

int connectionID =0;

int panPosition = 0;

boolean stillRunning = true;

ServerThread(Socket _socket) {
mySocket = _socket;

// trade the standard byte input stream for a fancier one that allows for more than just bytes
try {
din = new DataInputStream(mySocket.getInputStream());
dout = new DataOutputStream(mySocket.getOutputStream());
} catch (IOException e) {
System.out.println(“couldn’t get streams” + e);
}
}

public void run() {
try {

while (stillRunning) {
String header = null;
byte[] enclosure = null;

header = din.readUTF();

if (header == null) {
killMe();
break;
}

// protocol is type,size,who
String[] headerParts = header.split(“,”);
int size = Integer.parseInt(headerParts[1]);

enclosure = new byte[size];
din.readFully(enclosure);
header = headerParts[0] + “,” + headerParts[1] + “,” + headerParts[2] + “,” + headerParts[3] ;

tellEveryone(this,header, enclosure);
//System.out.println(“server got ” + header );
}
} catch (IOException e) {
killMe();
System.out.println(“Connection Lost”);

}

}

public void killMe() {
System.out.println(“Removing Connection”);
stillRunning = false;
removeMe(this);
}

public void setConnectionID(int _cn){
connectionID = _cn;
}

public int getConnectionID(){
return connectionID;
}

}

/**
* @author Dan O’Sullivan A thread that listens text and relays it to other people Try to add and introduction that asks for their name and then preppends name to all comments Try adding functionality where you can wisper to a specific person
*/
public class ServerSenderThread extends Thread {

DataOutputStream sendOut;
byte[] sendEnclosure;
String sendHeader;

ServerSenderThread(DataOutputStream _dout, String _header, byte[] _enclosure) {
sendOut = _dout;
sendHeader = _header;
sendEnclosure = _enclosure;
start();
}

public void run() {
try {
sendOut.writeUTF(sendHeader);
sendOut.flush();
if (sendEnclosure != null){
sendOut.write(sendEnclosure);
sendOut.flush();
}
} catch (IOException e) {
System.out.println(“Problem sending”);
}

}

}
}

Leave a Reply