View source for Wii - remote composer - Javi Lee

== Description ==
*Motivation

I wanted to explore the difference between the minds of men and women. I wanted to see what the males prefer over the preferences of the female and vice versa. In a way to explore the psychological development of the human brain and if gender has a factor to it. 

*Interactive paradigm

I wanted to take the physical space and turn it into experiment lab. The user will not be able to see that the program is tracking what he/she is looking at but the observers will be able to see. 

*Technical Description

I want to use Processing to display the tracks of where the eyeball will be looking at. It will display pictures and the participant will just look at the pictures. I wanted to put an eyeball tracker that will track where the eye is looking at. In this way it will track what the participants are looking at.

==Visualization==
*Functional Diagram
program code:

import java.util.ArrayList;

import processing.core.PApplet;
import processing.video.Capture;

public class ICU extends PApplet {
  final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3;
   final int KIM = 4;
   int item = JESSICA;
boolean usingMouse = false; //to use mouse for preso, if needed!

   Capture video;// regular processing libary
   int pupilThreshold = 100;
   int glintThreshold = 240;
   int edge = 50;
   int reach = 5;
   int minPupilArea = 75;
   long elapsedTime;
   float calibrationSlope;
   int calibrationIntercept;
   Rectangle pupil;
   Rectangle glint;
   CoordinatesTranslator calibrator;

PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; 
long lastVideo = 0;
 
//variables for easing
float crosshairLocationOnScreenX;
float crosshairLocationOnScreenY;
float targetX, targetY;
float easing = 0.05;
   boolean debug = true;
   PImage testImage;
   boolean leaveTrails;
   static public void main(String _args[]) {
     PApplet.main(new String[] {
       "tracking.Eyetrack"                  }
    );
   }
int num = 20;
// Images must be in the "data" directory to load correctly 
float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius;

public void setup() {
   size(600, 800);
   noStroke();
   smooth();
   //ellipseMode(RADIUS);
   //rectMode(CORNERS);
   println( Capture.list());
    video = new Capture(this, width, height, Capture.list()[4] );
     //video.settings();
     testImage = loadImage("mm.jpg");
     size(testImage.width,testImage.height);
     calibrator = new CoordinatesTranslator();
     image(testImage,0,0);

       smooth();
       
       img = loadImage("korean-girls-kang-su-yeon.jpg");
       imgJessica = loadImage("Jessica_alba.jpg");
       imgMaggie = loadImage("maggie_q_1.jpg");
       imgHyori = loadImage("lee_hyori.jpg");
       imgNamie = loadImage("namie_amuro_1.jpg");

       

     noCursor();

imageMode(CENTER);

}

public void draw() {
   background(225);
   drawTarget();
   stroke(225,0,0);
   noFill();
   //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX;
     if (abs(mouseX - mx) > 0.1) {
     mx = mx + (mouseX - mx) * easing;
   }
   if (abs(mouseY - my) > 0.1) {
     my = my + (mouseY- my) * easing;
   }

   mx = constrain(mx, inner, width - inner);
   my = constrain(my, inner, height - inner);
   //fill(76);
   //rect(edge, edge, width-edge, height-edge);
   fill(0);
   ellipse(mx, my, radius, radius);

   int which = frameCount % num;
   mx[which] = mouseX;
   my[which] = mouseY;

   for (int i = 0; i < num; i++) {
      //which+1 is the smallest (the oldest in the array)
     int index = (which+1 + i) % num;
     ellipse(mx[index], my[index], i, i); }
   if(abs(dx) > 1) {
     crosshairLocationOnScreenX += dx * easing;
   }

   //targetY = drawY;
   float dy = drawY - crosshairLocationOnScreenY;
   if(abs(dy) > 1) {
     crosshairLocationOnScreenY += dy * easing;
   }
   
if (video.available() && (millis() > lastVideo + 10)) {
   lastVideo = millis();
       video.read();
       long startTime = millis();
       elapsedTime = millis() - startTime;
       if (debug) {
         background(0);
         image(video, 0, 0);
       }
       else{
         if (leaveTrails == false) image(testImage,0,0);
          if (calibrating) { image(testImage,0,0); }
          if (pupil != null) {
            ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40);
            //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width));
            //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/
            //drawX = adjustedPoint[0];
            //drawY = adjustedPoint[1];*/
            //println(adjustedPoint[0]+", "+adjustedPoint[1]);
          }
       }
       pupil = findPupil();
       glint = null;
       if (pupil != null) glint = findGlint(pupil);
       //if the glint and the pupil are present
       if (glint != null && pupil != null) { // if we got both
         // find out where the pupil is relative to the glint
         int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2;
         int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2  ;
         //if the system is calibrating
         if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode
           Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at
           //draw the dots for the user to look at for calibration purposes
           placeToDrawTarget = placeToLook;
           if (placeToDrawTarget.x > 0 &&
calibrator.isCurrentlyCalibrating()) {
           stroke(0);
           fill(0);
           ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look
           stroke(255);
           fill(255);
           ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look
           stroke(40,40,255);
           fill(40,40,255);
           ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look
           stroke(255,0,0);
           fill(255,0,0);
           ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look
           stroke(255,255,0);
           fill(255,255,0);
           ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look
           }
           //CHANGE THIS TO ELMO

           calibrator.doCalibrationRoutine(true,rawX, rawY);
           println("calibrating");

         }
         //if the system is not calibrating,
         else {
           //use the calibrator to find out where the x,y from camera corolate to on image
           int[] adjustedPoint = calibrator.translate(rawX, rawY);
           println("Adjusted x" + adjustedPoint[0]  + " adjusted y" + adjustedPoint[1]);

           //if the mouse is not being used, draw the crosshairs at adjusted points
           if(!usingMouse) {
             drawX = adjustedPoint[0];
             drawY = adjustedPoint[1];
           }
         }

       }

     }

     }

   public Rectangle findPupil() {
     // /FIND THE PUPIL
     ArrayList boxes = new ArrayList();
     for (int row = edge; row < video.height - edge * 2; row++) {
       for (int col = edge; col < video.width - edge * 2; col++) {
         int offset = row * video.width + col;
         int thisPixel = video.pixels[offset];
         //look for dark things
         if (brightness(thisPixel) < pupilThreshold) {
           video.pixels[offset] = 0;
           // be pessimistic
           boolean foundAHome = false;
           // look throught the existing
           for (int i = 0; i < boxes.size(); i++) {
             Rectangle existingBox = (Rectangle) boxes.get(i);
             // is this spot in an existing box
             Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
             inflatedBox.grow(reach, reach); // widen it's reach
             if (inflatedBox.contains(col, row)) {
               existingBox.add(col, row);
               foundAHome = true; // no need to make a new one
               break; // no need to look through the rest of the boxes
             }
           }
           // if this does not belong to one of the existing boxes make a new one at this place
           if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0));
         }
       }
     }

     consolidate(boxes, 0, 0);

     // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER
     Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea);
     if (debug) {
       // show the the edges of the search
       fill(0, 0, 0, 0);
       stroke(0, 255, 0);
       rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge);
       // show all the pupil candidates
       stroke(0, 0, 0);
       for (int i = 0; i < boxes.size(); i++) {
         Rectangle thisBox = (Rectangle) boxes.get(i);
         rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
       }
       // show the winning pupil candidate in red
       if (pupil != null) {
         stroke(255, 0, 0);
         rect(pupil.x, pupil.y, pupil.width, pupil.height);
       }
     }
     return pupil;
   }

   public Rectangle findGlint(Rectangle _pupil) {
     // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL
     int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2);
     int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2);
     int glintTop = _pupil.y;
     int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);

     ArrayList glintsCandidates = new ArrayList();
     for (int row = glintTop; row < glintBottom; row++) {
       for (int col = glintEdgeL; col < glintEdgeR; col++) {
         int offset = row * video.width + col;
         int thisPixel = video.pixels[offset];
         //look for bright things
         if (brightness(thisPixel) > glintThreshold) {
           if (debug) video.pixels[offset] = 0;
           // be pessimistic
           boolean foundAHome = false;
           // look throught the existing
           for (int i = 0; i < glintsCandidates.size(); i++) {
             Rectangle existingBox = (Rectangle) glintsCandidates.get (i);
             // is this spot in an existing box
             Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
             inflatedBox.grow(reach, reach); // widen it's reach
             if (inflatedBox.contains(col, row)) {
               existingBox.add(col, row);
               foundAHome = true; // no need to make a new one
               break; // no need to look through the rest of the boxes
             }
           }
           // if this does not belong to one of the existing boxes make a new one at this place
           if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0));
         }
       }
     }
     // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL
     Rectangle glint = findClosestMostBigOne(glintsCandidates,
_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0);
     stroke(0, 0, 255);

     if (debug) {// show all the candidate
       // show the edges of the search for the glint
       stroke(0, 75, 200);
       rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop);
       for (int i = 0; i < glintsCandidates.size(); i++) {
         Rectangle thisBox = (Rectangle) glintsCandidates.get(i);
         rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
       } // show the winner in red
       if (glint != null) {
         stroke(255, 0, 0);
         rect(glint.x, glint.y, glint.width, glint.height);
       }
     }

     return glint;
   }

   public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) {
     // check every combination of shapes for overlap
     // make the repeat loop backwards so you delete off the bottom of the stack
     for (int i = _shapes.size() - 1; i > -1; i--) {
       // only check the ones up
       Rectangle shape1 = (Rectangle) _shapes.get(i);
       Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box
       inflatedShape1.grow(_consolidateReachX,
_consolidateReachY); // widen it's reach

       for (int j = i - 1; j > -1; j--) {
         Rectangle shape2 = (Rectangle) _shapes.get(j);
         if (inflatedShape1.intersects(shape2)) {
           shape1.add(shape2);
           // System.out.println("Remove" + j);
           _shapes.remove(j);
           break;
         }
       }
     }
   }

   public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) {
     if (_allRects.size() == 0) return null;
     int winner = 0;
     float closest = 1000;

     for (int i = 0; i < _allRects.size(); i++) {
       Rectangle thisRect = (Rectangle)  _allRects.get(i);
       if (thisRect.width * thisRect.height < _minArea) continue;
       float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2);
       if (thisDist < closest) {
         closest = thisDist;
         winner = i;
       }
     }
     return (Rectangle) _allRects.get(winner);
   }
   void stop() {
   }
}