Wii - remote composer - Javi Lee
From Robert-Depot
Description
- Motivation
I wanted to explore the difference between the minds of men and women. I wanted to see what the males prefer over the preferences of the female and vice versa. In a way to explore the psychological development of the human brain and if gender has a factor to it.
- Interactive paradigm
I wanted to take the physical space and turn it into experiment lab. The user will not be able to see that the program is tracking what he/she is looking at but the observers will be able to see.
- Technical Description
I want to use Processing to display the tracks of where the eyeball will be looking at. It will display pictures and the participant will just look at the pictures. I wanted to put an eyeball tracker that will track where the eye is looking at. In this way it will track what the participants are looking at.
Visualization
- Functional Diagram
program code:
import java.util.ArrayList;
import processing.core.PApplet; import processing.video.Capture;
public class ICU extends PApplet {
final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3; final int KIM = 4; int item = JESSICA;
boolean usingMouse = false; //to use mouse for preso, if needed!
Capture video;// regular processing libary int pupilThreshold = 100; int glintThreshold = 240; int edge = 50; int reach = 5; int minPupilArea = 75; long elapsedTime; float calibrationSlope; int calibrationIntercept; Rectangle pupil; Rectangle glint; CoordinatesTranslator calibrator;
PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; long lastVideo = 0;
//variables for easing float crosshairLocationOnScreenX; float crosshairLocationOnScreenY; float targetX, targetY; float easing = 0.05;
boolean debug = true;
PImage testImage;
boolean leaveTrails;
static public void main(String _args[]) {
PApplet.main(new String[] {
"tracking.Eyetrack" }
);
}
int num = 20; // Images must be in the "data" directory to load correctly float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius;
public void setup() {
size(600, 800);
noStroke();
smooth();
//ellipseMode(RADIUS);
//rectMode(CORNERS);
println( Capture.list());
video = new Capture(this, width, height, Capture.list()[4] );
//video.settings();
testImage = loadImage("mm.jpg");
size(testImage.width,testImage.height);
calibrator = new CoordinatesTranslator();
image(testImage,0,0);
smooth();
img = loadImage("korean-girls-kang-su-yeon.jpg");
imgJessica = loadImage("Jessica_alba.jpg");
imgMaggie = loadImage("maggie_q_1.jpg");
imgHyori = loadImage("lee_hyori.jpg");
imgNamie = loadImage("namie_amuro_1.jpg");
noCursor();
imageMode(CENTER);
}
public void draw() {
background(225);
drawTarget();
stroke(225,0,0);
noFill();
//draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX;
if (abs(mouseX - mx) > 0.1) {
mx = mx + (mouseX - mx) * easing;
}
if (abs(mouseY - my) > 0.1) {
my = my + (mouseY- my) * easing;
}
mx = constrain(mx, inner, width - inner); my = constrain(my, inner, height - inner); //fill(76); //rect(edge, edge, width-edge, height-edge); fill(0); ellipse(mx, my, radius, radius);
int which = frameCount % num; mx[which] = mouseX; my[which] = mouseY;
for (int i = 0; i < num; i++) {
//which+1 is the smallest (the oldest in the array)
int index = (which+1 + i) % num;
ellipse(mx[index], my[index], i, i); }
if(abs(dx) > 1) {
crosshairLocationOnScreenX += dx * easing;
}
//targetY = drawY;
float dy = drawY - crosshairLocationOnScreenY;
if(abs(dy) > 1) {
crosshairLocationOnScreenY += dy * easing;
}
if (video.available() && (millis() > lastVideo + 10)) {
lastVideo = millis();
video.read();
long startTime = millis();
elapsedTime = millis() - startTime;
if (debug) {
background(0);
image(video, 0, 0);
}
else{
if (leaveTrails == false) image(testImage,0,0);
if (calibrating) { image(testImage,0,0); }
if (pupil != null) {
ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40);
//drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width));
//drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/
//drawX = adjustedPoint[0];
//drawY = adjustedPoint[1];*/
//println(adjustedPoint[0]+", "+adjustedPoint[1]);
}
}
pupil = findPupil();
glint = null;
if (pupil != null) glint = findGlint(pupil);
//if the glint and the pupil are present
if (glint != null && pupil != null) { // if we got both
// find out where the pupil is relative to the glint
int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2;
int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2 ;
//if the system is calibrating
if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode
Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at
//draw the dots for the user to look at for calibration purposes
placeToDrawTarget = placeToLook;
if (placeToDrawTarget.x > 0 &&
calibrator.isCurrentlyCalibrating()) {
stroke(0);
fill(0);
ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look
stroke(255);
fill(255);
ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look
stroke(40,40,255);
fill(40,40,255);
ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look
stroke(255,0,0);
fill(255,0,0);
ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look
stroke(255,255,0);
fill(255,255,0);
ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look
}
//CHANGE THIS TO ELMO
calibrator.doCalibrationRoutine(true,rawX, rawY);
println("calibrating");
}
//if the system is not calibrating,
else {
//use the calibrator to find out where the x,y from camera corolate to on image
int[] adjustedPoint = calibrator.translate(rawX, rawY);
println("Adjusted x" + adjustedPoint[0] + " adjusted y" + adjustedPoint[1]);
//if the mouse is not being used, draw the crosshairs at adjusted points
if(!usingMouse) {
drawX = adjustedPoint[0];
drawY = adjustedPoint[1];
}
}
}
}
}
public Rectangle findPupil() {
// /FIND THE PUPIL
ArrayList boxes = new ArrayList();
for (int row = edge; row < video.height - edge * 2; row++) {
for (int col = edge; col < video.width - edge * 2; col++) {
int offset = row * video.width + col;
int thisPixel = video.pixels[offset];
//look for dark things
if (brightness(thisPixel) < pupilThreshold) {
video.pixels[offset] = 0;
// be pessimistic
boolean foundAHome = false;
// look throught the existing
for (int i = 0; i < boxes.size(); i++) {
Rectangle existingBox = (Rectangle) boxes.get(i);
// is this spot in an existing box
Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
inflatedBox.grow(reach, reach); // widen it's reach
if (inflatedBox.contains(col, row)) {
existingBox.add(col, row);
foundAHome = true; // no need to make a new one
break; // no need to look through the rest of the boxes
}
}
// if this does not belong to one of the existing boxes make a new one at this place
if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0));
}
}
}
consolidate(boxes, 0, 0);
// OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER
Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea);
if (debug) {
// show the the edges of the search
fill(0, 0, 0, 0);
stroke(0, 255, 0);
rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge);
// show all the pupil candidates
stroke(0, 0, 0);
for (int i = 0; i < boxes.size(); i++) {
Rectangle thisBox = (Rectangle) boxes.get(i);
rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
}
// show the winning pupil candidate in red
if (pupil != null) {
stroke(255, 0, 0);
rect(pupil.x, pupil.y, pupil.width, pupil.height);
}
}
return pupil;
}
public Rectangle findGlint(Rectangle _pupil) {
// ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL
int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2);
int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2);
int glintTop = _pupil.y;
int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);
ArrayList glintsCandidates = new ArrayList();
for (int row = glintTop; row < glintBottom; row++) {
for (int col = glintEdgeL; col < glintEdgeR; col++) {
int offset = row * video.width + col;
int thisPixel = video.pixels[offset];
//look for bright things
if (brightness(thisPixel) > glintThreshold) {
if (debug) video.pixels[offset] = 0;
// be pessimistic
boolean foundAHome = false;
// look throught the existing
for (int i = 0; i < glintsCandidates.size(); i++) {
Rectangle existingBox = (Rectangle) glintsCandidates.get (i);
// is this spot in an existing box
Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
inflatedBox.grow(reach, reach); // widen it's reach
if (inflatedBox.contains(col, row)) {
existingBox.add(col, row);
foundAHome = true; // no need to make a new one
break; // no need to look through the rest of the boxes
}
}
// if this does not belong to one of the existing boxes make a new one at this place
if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0));
}
}
}
// FIND THE GLINT THAT IS CLOSEST TO THE PUPIL
Rectangle glint = findClosestMostBigOne(glintsCandidates,
_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0);
stroke(0, 0, 255);
if (debug) {// show all the candidate
// show the edges of the search for the glint
stroke(0, 75, 200);
rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop);
for (int i = 0; i < glintsCandidates.size(); i++) {
Rectangle thisBox = (Rectangle) glintsCandidates.get(i);
rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
} // show the winner in red
if (glint != null) {
stroke(255, 0, 0);
rect(glint.x, glint.y, glint.width, glint.height);
}
}
return glint; }
public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) {
// check every combination of shapes for overlap
// make the repeat loop backwards so you delete off the bottom of the stack
for (int i = _shapes.size() - 1; i > -1; i--) {
// only check the ones up
Rectangle shape1 = (Rectangle) _shapes.get(i);
Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box
inflatedShape1.grow(_consolidateReachX,
_consolidateReachY); // widen it's reach
for (int j = i - 1; j > -1; j--) {
Rectangle shape2 = (Rectangle) _shapes.get(j);
if (inflatedShape1.intersects(shape2)) {
shape1.add(shape2);
// System.out.println("Remove" + j);
_shapes.remove(j);
break;
}
}
}
}
public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) {
if (_allRects.size() == 0) return null;
int winner = 0;
float closest = 1000;
for (int i = 0; i < _allRects.size(); i++) {
Rectangle thisRect = (Rectangle) _allRects.get(i);
if (thisRect.width * thisRect.height < _minArea) continue;
float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2);
if (thisDist < closest) {
closest = thisDist;
winner = i;
}
}
return (Rectangle) _allRects.get(winner);
}
void stop() {
}
}
