As people stand in front of a mirror, this applications uses face detection technology to find the faces in the video and then display them with just the faces switched. Each person looks normal except they have someone elses eyes, nose and mouth. This far I go before. In this excersize I wanted it to work for a crowd standing in front of the large screen display at IAC, Gehry building. This required that the faces be shared across multiple application so today I did the networking for that. If I had more time I would add an oval alpha mask so the faces blend better.  This was done as a Monday project in the 7 projects in 7 days (5 in 5 for me) festival at ITP.

Tech: I used the OpenCV processing libraries for the face detection. I used IP cameras for the video capture. Below is the code:

Client:

import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UTFDataFormatException;
import java.net.Socket;
import java.net.UnknownHostException;

import pFaceDetect.PFaceDetect;
import processing.core.PApplet;
import processing.core.PImage;
import processing.net.Server;
import vxp.CaptureAxisCamera;

import com.sun.image.codec.jpeg.ImageFormatException;
import com.sun.image.codec.jpeg.JPEGCodec;
import com.sun.image.codec.jpeg.JPEGEncodeParam;
import com.sun.image.codec.jpeg.JPEGImageDecoder;
import com.sun.image.codec.jpeg.JPEGImageEncoder;
import com.sun.image.codec.jpeg.TruncatedFileException;

public class FaceLift extends PApplet {

Server myServer;

int endOfMessageChar = 10;

Socket mySocket;

int socketPort = 9001;

String ip = “localhost”;

CaptureAxisCamera video;

boolean reply = true;

static String[] args;

int otherx, othery, otherw, otherh;

PImage other;

int myConnectionNumber = -1;

// PImage img;

static public void main(String _args[]) {
PApplet.main(new String[] { “FaceLift” });
}

PFaceDetect face;

public void setup() {
size(320, 240);
video = new CaptureAxisCamera(this, “128.122.151.189”, width, height, false);

face = new PFaceDetect(this, width, height, “haarcascade_frontalface_default.xml”);
frameRate(15);

noFill();
try {
mySocket = new Socket(ip, socketPort);
} catch (UnknownHostException e) {
println(“Could not find host address”);
} catch (IOException e) {
println(“Could not connect to host”);
}
println(“Connecting as client at ” + socketPort + ” on ” + ip);
new TeleListener(mySocket);

}

void drawFace() {
int[][] res = face.getFaces();
if (res.length > 0) {
for (int i = 0; i < res.length; i++) {
int x = res[i][0];
int y = res[i][1];
int w = res[i][2];
int h = res[i][3];
rect(x, y, w, h);
}
}
}

public void draw() {
background(0);
video.read();
//if (video.available()) {
face.findFaces(video);
image(video, 0, 0);
drawFace();
if (other != null) {

PImage img = createImage(otherw, otherh, ARGB);
int centerx = width/2;
int centery = height/2;
for(int y=0; y < img.height; y++) {
for(int x=0; x < img.width; x++) {
int alpha = (int) (100*dist(x,centerx,y,centery)/width/2);
img.pixels[y*otherw+x] = color(0, 90, 102, alpha);
}

}

other.mask(img);
/*BufferedImage oval = new BufferedImage(otherw, otherh, BufferedImage.TYPE_INT_ARGB);
Graphics2D ovalGraphics = oval.createGraphics();
ovalGraphics.setBackground(new Color(255, 0, 0, 0));
ovalGraphics.clearRect(0, 0, otherw, otherh);
ovalGraphics.setColor(new Color(255, 255, 255, 255));
ovalGraphics.fillOval(0, 0, otherw, otherh);
image(new PImage(oval), otherx, othery, otherw, otherh);
*/
image(other, otherx, othery, otherw, otherh);
}
//sendPicture();
//}
}

public void sendPicture() {
if (mySocket != null && video != null) {

int[][] res = face.getFaces();
if (res.length > 0) {
//println(“Sendit”);
otherx = res[0][0];

othery = res[0][1];
otherw = res[0][2];
otherh = res[0][3];
// rect(x, y, w, h);
BufferedImage bi = video.getImage().getSubimage(otherx, othery, otherw, otherh);
new Sender(mySocket, bi);
/*
* for (int i = res.length – 1; i > res.length – 2; i–) { int x = res[i][0]; int y = res[i][1]; int w = res[i][2]; int h = res[i][3]; rect(x, y, w, h); BufferedImage bi = video.getImage().getSubimage(x, x, w, h); new Sender(mySocket, bi); }
*/
}
}

}

public void mousePressed() {
sendPicture();
}

public void incoming(PImage _img, int _connectionNumber) {
//println(“INcoming”);
if (_img != null) other = _img;

}

public class TeleListener extends Thread {

DataInputStream dis;

DataOutputStream dout;

int connectionNumber = 0;

TeleListener(Socket _c) {

try {
dis = new DataInputStream(_c.getInputStream());
dout = new DataOutputStream(_c.getOutputStream());
} catch (IOException e) {
println(“Couldn’t Connect”);
}

start();
}

public void run() {

while (true) {
int size = 0;
int connectionNumber = 0;

try {
String header = null;
try {

header = dis.readUTF();
} catch (UTFDataFormatException e) {
System.out.println(” UTFDataFormatException exeption “);
continue;
}
if (header == null) break;
try {
String[] messageParts = header.split(“,”);
size = Integer.parseInt(messageParts[1]);

} catch (RuntimeException e) {
System.out.println(” RuntimeException exeption “);
continue;
}

if (size == 0) continue;

// make a buffer that size to accept the incoming bytes
byte[] incoming = new byte[size];
dis.readFully(incoming);
PImage myImage = null;
myImage = byteArrayToPImage(incoming);

incoming(myImage, connectionNumber);
// println(connectionNumber + ” Client got” + size);
} catch (NumberFormatException e) {
System.out.println(“Text to number problem.”);
break;
} catch (IOException e) {
System.out.println(“Nothing came in.”);
break;
}

}
}
}

public class Sender extends Thread {
DataOutputStream dos;

DataInputStream dis;

BufferedImage img;

Sender(Socket _c, BufferedImage _image) {
try {
dos = new DataOutputStream(_c.getOutputStream());
dis = new DataInputStream(_c.getInputStream());
} catch (IOException e) {
println(“Couldn’t Connect”);
}
img = _image;
start();
}

public void run() {

byte[] imageInBytes = imageToByteArray(img);
// tell them how many bytes to expect
String outSize = String.valueOf(imageInBytes.length);
String header = “enclosure,” + outSize;

try {
DataOutputStream dos = new DataOutputStream(mySocket.getOutputStream());
dos.writeUTF(header);
dos.flush();
dos.write(imageInBytes);
dos.flush();
} catch (IOException e) {
e.printStackTrace();
}

}

}

public byte[] imageToByteArray(BufferedImage _image) {

// bit if a hack to convert PImage to a Buffered Image, draw it on the buffered image
// ((BufferedImage) image).getRGB(x, y, w, h, img.pixels, offset, img.width);

// BufferedImage bi = new BufferedImage(_image.width, _image.height, BufferedImage.TYPE_INT_RGB);

// bi.setRGB(0, 0, _image.width, _image.height, _image.pixels, 0, _image.width);
// compress and turn into a byte array
// okay just to copy and paste this method

ByteArrayOutputStream baOut = new ByteArrayOutputStream();
JPEGImageEncoder encoder = JPEGCodec.createJPEGEncoder(baOut);
JPEGEncodeParam param = encoder.getDefaultJPEGEncodeParam(_image);
param.setQuality(0.9f, false);
encoder.setJPEGEncodeParam(param);
try {
encoder.encode(_image);
baOut.flush();
} catch (ImageFormatException e) {
System.out.println(“could not form image” + e);
} catch (IOException e) {
System.out.println(“could not encode image” + e);
}

return baOut.toByteArray();
}

public PImage byteArrayToPImage(byte[] _pic) {
// decompress into a buffferedImage
// okay just to copy and paste this method
ByteArrayInputStream in = new ByteArrayInputStream((byte[]) _pic);
JPEGImageDecoder decoder = JPEGCodec.createJPEGDecoder(in);
BufferedImage snapShot = null;
try {
snapShot = decoder.decodeAsBufferedImage();
} catch (ImageFormatException e) {
System.out.println(“Could not make array into a jpeg. ” + _pic.length);

} catch (TruncatedFileException e) {
System.out.println(“Truncated File” + _pic.length);
} catch (IOException e) {
System.out.println(“IO Making file” + _pic.length);
}

if (snapShot == null) return null;
return new PImage(snapShot);
}

void getParams() {
if (args != null) { // running as an application
if (args.length > 0) socketPort = Integer.parseInt(args[0]);
if (args.length > 1) {
ip = args[1];

}
} else { // running as an applet

String isItThere = getParameter(“socketPort”);
if (isItThere != null) socketPort = Integer.parseInt(isItThere);
isItThere = getParameter(“ip”);
if (isItThere != null) ip = isItThere;
}
}

}

Server:

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.ArrayList;

/**
* @author Dan O’Sullivan
*
* Allows conections to come in and spawns thread to take care of them
*/
public class FaceServer {
int lastConnectionNumber = 0;

ArrayList allConnections = new ArrayList();

public static void main(String[] args) {
new FaceServer();
}

FaceServer() {
ServerSocket frontDoor;
int portNum = 9001;
try {
frontDoor = new ServerSocket(portNum);
System.out.println(“Set up a door at ” + InetAddress.getLocalHost() + ” ” + frontDoor.getLocalPort());
while (true) {
System.out.println(“Waiting for a new connection”);
Socket newSocket = frontDoor.accept();// sits here and waits for // someone to knock on the door
System.out.println(newSocket.getRemoteSocketAddress() + ” knocked on the door and I let them in”);
// farm it out immediately to another thread
ServerThread newServerThread = new ServerThread(newSocket);
allConnections.add(newServerThread); // add it to the list of connections
newServerThread.setConnectionID(lastConnectionNumber);
lastConnectionNumber++;
newServerThread.start();// this calls the run method in the thread
}
} catch (IOException e) {
System.out.println(“Had a problem making a front door” + e);
}
}

synchronized public void tellNextPerson(ServerThread _me, String _header, byte[] _enclosure) {
ServerThread nextPerson = null;

for (int i = 0; i < allConnections.size(); i++) {

ServerThread thisConnection = (ServerThread) allConnections.get(i);
if (thisConnection == _me) { //don’t send back to yourself
int nextIndex = i+1;
if (nextIndex >= allConnections.size()) nextIndex = 0;
nextPerson = (ServerThread) allConnections.get(nextIndex);
String outHeader = _header;
// System.out.println(“Server Distributes ” + outHeader);
new ServerSenderThread(nextPerson.dout,outHeader,_enclosure);
break;
}
}

}

synchronized public void removeMe(ServerThread _which) {
allConnections.remove(_which);

}

/**
* @author Dan O’Sullivan A thread that listens text and relays it to other people Try to add and introduction that asks for their name and then preppends name to all comments Try adding functionality where you can wisper to a specific person
*/
public class ServerThread extends Thread {
Socket mySocket;

DataInputStream din;

public DataOutputStream dout;

int connectionID =0;

int panPosition = 0;

boolean stillRunning = true;

ServerThread(Socket _socket) {
mySocket = _socket;

// trade the standard byte input stream for a fancier one that allows for more than just bytes
try {
din = new DataInputStream(mySocket.getInputStream());
dout = new DataOutputStream(mySocket.getOutputStream());
} catch (IOException e) {
System.out.println(“couldn’t get streams” + e);
}
}

public void run() {
try {

while (stillRunning) {
String header = null;
byte[] enclosure = null;

header = din.readUTF();

if (header == null) {
killMe();
break;
}

// protocol is type,size,who
String[] headerParts = header.split(“,”);
int size = Integer.parseInt(headerParts[1]);

enclosure = new byte[size];
din.readFully(enclosure);
header = headerParts[0] + “,” + headerParts[1] ;

tellNextPerson(this,header, enclosure);
//System.out.println(“server got ” + header );
}
} catch (IOException e) {
killMe();
System.out.println(“Connection Lost”);

}

}

public void killMe() {
System.out.println(“Removing Connection”);
stillRunning = false;
removeMe(this);
}

public void setConnectionID(int _cn){
connectionID = _cn;
}

public int getConnectionID(){
return connectionID;
}

}

/**
* @author Dan O’Sullivan A thread that listens text and relays it to other people Try to add and introduction that asks for their name and then preppends name to all comments Try adding functionality where you can wisper to a specific person
*/
public class ServerSenderThread extends Thread {

DataOutputStream sendOut;
byte[] sendEnclosure;
String sendHeader;

ServerSenderThread(DataOutputStream _dout, String _header, byte[] _enclosure) {
sendOut = _dout;
sendHeader = _header;
sendEnclosure = _enclosure;
start();
}

public void run() {
try {
sendOut.writeUTF(sendHeader);
sendOut.flush();
if (sendEnclosure != null){
sendOut.write(sendEnclosure);
sendOut.flush();
}
} catch (IOException e) {
System.out.println(“Problem sending”);
}

}

}
}