Difference between revisions of "Eye Motion Experiment - Javi Lee"

Latest revision as of 13:18, 4 June 2010

Description

• Motivation

My main motivation for this project was to experiment and document on the processes of the human mind differentiating between males and females. I wanted to see what each gender would specifically look at when shown an image. My hypothesis would be that when shown an image of a sexy female, the males would naturally look at the breast and the butt, whereas the females would look more at the face and the abs. I wanted to see if males and females minds are the same or if they were indeed different.

In order to see how this experiment would work, I wanted to hook up each participant to an eye tracking webcam. This webcam would track the iris of the eye and display that path on the screen. This track will be clear and visible for the audience standing behind the participant to see. The participant will only see the images but will not be able to see the screen where his eyes will be tracking. I wanted to display an array of images, not limited to people, and see what aspect of the images each gender concentrates on.

• Technical Description

I wanted to use Processing to create a code that will track the eyeball. This code will also draw the path that the eyeball takes. In order to track the eyeball, I wanted to create some kind of glasses that is connected to a webcam that will track the iris of the eyeball.

Unfortunately this one did not work.

Documentation

import java.util.ArrayList;

import processing.core.PApplet; import processing.video.Capture;

public class ICU extends PApplet {

 final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3;
  final int KIM = 4;
  int item = JESSICA;

boolean usingMouse = false; //to use mouse for preso, if needed!

  Capture video;// regular processing libary
  int pupilThreshold = 100;
  int glintThreshold = 240;
  int edge = 50;
  int reach = 5;
  int minPupilArea = 75;
  long elapsedTime;
  float calibrationSlope;
  int calibrationIntercept;
  Rectangle pupil;
  Rectangle glint;
  CoordinatesTranslator calibrator;

PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; long lastVideo = 0;

//variables for easing float crosshairLocationOnScreenX; float crosshairLocationOnScreenY; float targetX, targetY; float easing = 0.05;

  boolean debug = true;
  PImage testImage;
  boolean leaveTrails;
  static public void main(String _args[]) {
    PApplet.main(new String[] {
      "tracking.Eyetrack"                  }
   );
  }

int num = 20; // Images must be in the "data" directory to load correctly float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius;

public void setup() {

  size(600, 800);
  noStroke();
  smooth();
  //ellipseMode(RADIUS);
  //rectMode(CORNERS);
  println( Capture.list());
   video = new Capture(this, width, height, Capture.list()[4] );
    //video.settings();
    testImage = loadImage("mm.jpg");
    size(testImage.width,testImage.height);
    calibrator = new CoordinatesTranslator();
    image(testImage,0,0);

      smooth();
      
      img = loadImage("korean-girls-kang-su-yeon.jpg");
      imgJessica = loadImage("Jessica_alba.jpg");
      imgMaggie = loadImage("maggie_q_1.jpg");
      imgHyori = loadImage("lee_hyori.jpg");
      imgNamie = loadImage("namie_amuro_1.jpg");

    noCursor();

imageMode(CENTER);

}

public void draw() {

  background(225);
  drawTarget();
  stroke(225,0,0);
  noFill();
  //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX;
    if (abs(mouseX - mx) > 0.1) {
    mx = mx + (mouseX - mx) * easing;
  }
  if (abs(mouseY - my) > 0.1) {
    my = my + (mouseY- my) * easing;
  }

  mx = constrain(mx, inner, width - inner);
  my = constrain(my, inner, height - inner);
  //fill(76);
  //rect(edge, edge, width-edge, height-edge);
  fill(0);
  ellipse(mx, my, radius, radius);

  int which = frameCount % num;
  mx[which] = mouseX;
  my[which] = mouseY;

  for (int i = 0; i < num; i++) {
     //which+1 is the smallest (the oldest in the array)
    int index = (which+1 + i) % num;
    ellipse(mx[index], my[index], i, i); }
  if(abs(dx) > 1) {
    crosshairLocationOnScreenX += dx * easing;
  }

  //targetY = drawY;
  float dy = drawY - crosshairLocationOnScreenY;
  if(abs(dy) > 1) {
    crosshairLocationOnScreenY += dy * easing;
  }
  

if (video.available() && (millis() > lastVideo + 10)) {

  lastVideo = millis();
      video.read();
      long startTime = millis();
      elapsedTime = millis() - startTime;
      if (debug) {
        background(0);
        image(video, 0, 0);
      }
      else{
        if (leaveTrails == false) image(testImage,0,0);
         if (calibrating) { image(testImage,0,0); }
         if (pupil != null) {
           ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40);
           //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width));
           //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/
           //drawX = adjustedPoint[0];
           //drawY = adjustedPoint[1];*/
           //println(adjustedPoint[0]+", "+adjustedPoint[1]);
         }
      }
      pupil = findPupil();
      glint = null;
      if (pupil != null) glint = findGlint(pupil);
      //if the glint and the pupil are present
      if (glint != null && pupil != null) { // if we got both
        // find out where the pupil is relative to the glint
        int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2;
        int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2  ;
        //if the system is calibrating
        if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode
          Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at
          //draw the dots for the user to look at for calibration purposes
          placeToDrawTarget = placeToLook;
          if (placeToDrawTarget.x > 0 &&

calibrator.isCurrentlyCalibrating()) {

          stroke(0);
          fill(0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look
          stroke(255);
          fill(255);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look
          stroke(40,40,255);
          fill(40,40,255);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look
          stroke(255,0,0);
          fill(255,0,0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look
          stroke(255,255,0);
          fill(255,255,0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look
          }
          //CHANGE THIS TO ELMO

          calibrator.doCalibrationRoutine(true,rawX, rawY);
          println("calibrating");

        }
        //if the system is not calibrating,
        else {
          //use the calibrator to find out where the x,y from camera corolate to on image
          int[] adjustedPoint = calibrator.translate(rawX, rawY);
          println("Adjusted x" + adjustedPoint[0]  + " adjusted y" + adjustedPoint[1]);

          //if the mouse is not being used, draw the crosshairs at adjusted points
          if(!usingMouse) {
            drawX = adjustedPoint[0];
            drawY = adjustedPoint[1];
          }
        }

      }

    }

    }

  public Rectangle findPupil() {
    // /FIND THE PUPIL
    ArrayList boxes = new ArrayList();
    for (int row = edge; row < video.height - edge * 2; row++) {
      for (int col = edge; col < video.width - edge * 2; col++) {
        int offset = row * video.width + col;
        int thisPixel = video.pixels[offset];
        //look for dark things
        if (brightness(thisPixel) < pupilThreshold) {
          video.pixels[offset] = 0;
          // be pessimistic
          boolean foundAHome = false;
          // look throught the existing
          for (int i = 0; i < boxes.size(); i++) {
            Rectangle existingBox = (Rectangle) boxes.get(i);
            // is this spot in an existing box
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
            inflatedBox.grow(reach, reach); // widen it's reach
            if (inflatedBox.contains(col, row)) {
              existingBox.add(col, row);
              foundAHome = true; // no need to make a new one
              break; // no need to look through the rest of the boxes
            }
          }
          // if this does not belong to one of the existing boxes make a new one at this place
          if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0));
        }
      }
    }

    consolidate(boxes, 0, 0);

    // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER
    Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea);
    if (debug) {
      // show the the edges of the search
      fill(0, 0, 0, 0);
      stroke(0, 255, 0);
      rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge);
      // show all the pupil candidates
      stroke(0, 0, 0);
      for (int i = 0; i < boxes.size(); i++) {
        Rectangle thisBox = (Rectangle) boxes.get(i);
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
      }
      // show the winning pupil candidate in red
      if (pupil != null) {
        stroke(255, 0, 0);
        rect(pupil.x, pupil.y, pupil.width, pupil.height);
      }
    }
    return pupil;
  }

  public Rectangle findGlint(Rectangle _pupil) {
    // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL
    int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2);
    int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2);
    int glintTop = _pupil.y;
    int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);

    ArrayList glintsCandidates = new ArrayList();
    for (int row = glintTop; row < glintBottom; row++) {
      for (int col = glintEdgeL; col < glintEdgeR; col++) {
        int offset = row * video.width + col;
        int thisPixel = video.pixels[offset];
        //look for bright things
        if (brightness(thisPixel) > glintThreshold) {
          if (debug) video.pixels[offset] = 0;
          // be pessimistic
          boolean foundAHome = false;
          // look throught the existing
          for (int i = 0; i < glintsCandidates.size(); i++) {
            Rectangle existingBox = (Rectangle) glintsCandidates.get (i);
            // is this spot in an existing box
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
            inflatedBox.grow(reach, reach); // widen it's reach
            if (inflatedBox.contains(col, row)) {
              existingBox.add(col, row);
              foundAHome = true; // no need to make a new one
              break; // no need to look through the rest of the boxes
            }
          }
          // if this does not belong to one of the existing boxes make a new one at this place
          if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0));
        }
      }
    }
    // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL
    Rectangle glint = findClosestMostBigOne(glintsCandidates,

_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0);

    stroke(0, 0, 255);

    if (debug) {// show all the candidate
      // show the edges of the search for the glint
      stroke(0, 75, 200);
      rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop);
      for (int i = 0; i < glintsCandidates.size(); i++) {
        Rectangle thisBox = (Rectangle) glintsCandidates.get(i);
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
      } // show the winner in red
      if (glint != null) {
        stroke(255, 0, 0);
        rect(glint.x, glint.y, glint.width, glint.height);
      }
    }

    return glint;
  }

  public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) {
    // check every combination of shapes for overlap
    // make the repeat loop backwards so you delete off the bottom of the stack
    for (int i = _shapes.size() - 1; i > -1; i--) {
      // only check the ones up
      Rectangle shape1 = (Rectangle) _shapes.get(i);
      Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box
      inflatedShape1.grow(_consolidateReachX,

_consolidateReachY); // widen it's reach

      for (int j = i - 1; j > -1; j--) {
        Rectangle shape2 = (Rectangle) _shapes.get(j);
        if (inflatedShape1.intersects(shape2)) {
          shape1.add(shape2);
          // System.out.println("Remove" + j);
          _shapes.remove(j);
          break;
        }
      }
    }
  }

  public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) {
    if (_allRects.size() == 0) return null;
    int winner = 0;
    float closest = 1000;

    for (int i = 0; i < _allRects.size(); i++) {
      Rectangle thisRect = (Rectangle)  _allRects.get(i);
      if (thisRect.width * thisRect.height < _minArea) continue;
      float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2);
      if (thisDist < closest) {
        closest = thisDist;
        winner = i;
      }
    }
    return (Rectangle) _allRects.get(winner);
  }
  void stop() {
  }

}