Wii - remote composer - Javi Lee

From Robert-Depot
Jump to: navigation, search

Description

  • Motivation

I wanted to explore the difference between the minds of men and women. I wanted to see what the males prefer over the preferences of the female and vice versa. In a way to explore the psychological development of the human brain and if gender has a factor to it.

  • Interactive paradigm

I wanted to take the physical space and turn it into experiment lab. The user will not be able to see that the program is tracking what he/she is looking at but the observers will be able to see.

  • Technical Description

I want to use Processing to display the tracks of where the eyeball will be looking at. It will display pictures and the participant will just look at the pictures. I wanted to put an eyeball tracker that will track where the eye is looking at. In this way it will track what the participants are looking at.

Visualization

  • Functional Diagram

program code:

import java.util.ArrayList;

import processing.core.PApplet; import processing.video.Capture;

public class ICU extends PApplet {

 final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3;
  final int KIM = 4;
  int item = JESSICA;

boolean usingMouse = false; //to use mouse for preso, if needed!

  Capture video;// regular processing libary
  int pupilThreshold = 100;
  int glintThreshold = 240;
  int edge = 50;
  int reach = 5;
  int minPupilArea = 75;
  long elapsedTime;
  float calibrationSlope;
  int calibrationIntercept;
  Rectangle pupil;
  Rectangle glint;
  CoordinatesTranslator calibrator;

PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; long lastVideo = 0;

//variables for easing float crosshairLocationOnScreenX; float crosshairLocationOnScreenY; float targetX, targetY; float easing = 0.05;

  boolean debug = true;
  PImage testImage;
  boolean leaveTrails;
  static public void main(String _args[]) {
    PApplet.main(new String[] {
      "tracking.Eyetrack"                  }
   );
  }

int num = 20; // Images must be in the "data" directory to load correctly float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius;

public void setup() {

  size(600, 800);
  noStroke();
  smooth();
  //ellipseMode(RADIUS);
  //rectMode(CORNERS);
  println( Capture.list());
   video = new Capture(this, width, height, Capture.list()[4] );
    //video.settings();
    testImage = loadImage("mm.jpg");
    size(testImage.width,testImage.height);
    calibrator = new CoordinatesTranslator();
    image(testImage,0,0);
      smooth();
      
      img = loadImage("korean-girls-kang-su-yeon.jpg");
      imgJessica = loadImage("Jessica_alba.jpg");
      imgMaggie = loadImage("maggie_q_1.jpg");
      imgHyori = loadImage("lee_hyori.jpg");
      imgNamie = loadImage("namie_amuro_1.jpg");


    noCursor();

imageMode(CENTER);

}

public void draw() {

  background(225);
  drawTarget();
  stroke(225,0,0);
  noFill();
  //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX;
    if (abs(mouseX - mx) > 0.1) {
    mx = mx + (mouseX - mx) * easing;
  }
  if (abs(mouseY - my) > 0.1) {
    my = my + (mouseY- my) * easing;
  }
  mx = constrain(mx, inner, width - inner);
  my = constrain(my, inner, height - inner);
  //fill(76);
  //rect(edge, edge, width-edge, height-edge);
  fill(0);
  ellipse(mx, my, radius, radius);
  int which = frameCount % num;
  mx[which] = mouseX;
  my[which] = mouseY;
  for (int i = 0; i < num; i++) {
     //which+1 is the smallest (the oldest in the array)
    int index = (which+1 + i) % num;
    ellipse(mx[index], my[index], i, i); }
  if(abs(dx) > 1) {
    crosshairLocationOnScreenX += dx * easing;
  }
  //targetY = drawY;
  float dy = drawY - crosshairLocationOnScreenY;
  if(abs(dy) > 1) {
    crosshairLocationOnScreenY += dy * easing;
  }
  

if (video.available() && (millis() > lastVideo + 10)) {

  lastVideo = millis();
      video.read();
      long startTime = millis();
      elapsedTime = millis() - startTime;
      if (debug) {
        background(0);
        image(video, 0, 0);
      }
      else{
        if (leaveTrails == false) image(testImage,0,0);
         if (calibrating) { image(testImage,0,0); }
         if (pupil != null) {
           ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40);
           //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width));
           //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/
           //drawX = adjustedPoint[0];
           //drawY = adjustedPoint[1];*/
           //println(adjustedPoint[0]+", "+adjustedPoint[1]);
         }
      }
      pupil = findPupil();
      glint = null;
      if (pupil != null) glint = findGlint(pupil);
      //if the glint and the pupil are present
      if (glint != null && pupil != null) { // if we got both
        // find out where the pupil is relative to the glint
        int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2;
        int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2  ;
        //if the system is calibrating
        if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode
          Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at
          //draw the dots for the user to look at for calibration purposes
          placeToDrawTarget = placeToLook;
          if (placeToDrawTarget.x > 0 &&

calibrator.isCurrentlyCalibrating()) {

          stroke(0);
          fill(0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look
          stroke(255);
          fill(255);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look
          stroke(40,40,255);
          fill(40,40,255);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look
          stroke(255,0,0);
          fill(255,0,0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look
          stroke(255,255,0);
          fill(255,255,0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look
          }
          //CHANGE THIS TO ELMO
          calibrator.doCalibrationRoutine(true,rawX, rawY);
          println("calibrating");
        }
        //if the system is not calibrating,
        else {
          //use the calibrator to find out where the x,y from camera corolate to on image
          int[] adjustedPoint = calibrator.translate(rawX, rawY);
          println("Adjusted x" + adjustedPoint[0]  + " adjusted y" + adjustedPoint[1]);
          //if the mouse is not being used, draw the crosshairs at adjusted points
          if(!usingMouse) {
            drawX = adjustedPoint[0];
            drawY = adjustedPoint[1];
          }
        }
      }
    }
    }
  public Rectangle findPupil() {
    // /FIND THE PUPIL
    ArrayList boxes = new ArrayList();
    for (int row = edge; row < video.height - edge * 2; row++) {
      for (int col = edge; col < video.width - edge * 2; col++) {
        int offset = row * video.width + col;
        int thisPixel = video.pixels[offset];
        //look for dark things
        if (brightness(thisPixel) < pupilThreshold) {
          video.pixels[offset] = 0;
          // be pessimistic
          boolean foundAHome = false;
          // look throught the existing
          for (int i = 0; i < boxes.size(); i++) {
            Rectangle existingBox = (Rectangle) boxes.get(i);
            // is this spot in an existing box
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
            inflatedBox.grow(reach, reach); // widen it's reach
            if (inflatedBox.contains(col, row)) {
              existingBox.add(col, row);
              foundAHome = true; // no need to make a new one
              break; // no need to look through the rest of the boxes
            }
          }
          // if this does not belong to one of the existing boxes make a new one at this place
          if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0));
        }
      }
    }
    consolidate(boxes, 0, 0);
    // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER
    Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea);
    if (debug) {
      // show the the edges of the search
      fill(0, 0, 0, 0);
      stroke(0, 255, 0);
      rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge);
      // show all the pupil candidates
      stroke(0, 0, 0);
      for (int i = 0; i < boxes.size(); i++) {
        Rectangle thisBox = (Rectangle) boxes.get(i);
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
      }
      // show the winning pupil candidate in red
      if (pupil != null) {
        stroke(255, 0, 0);
        rect(pupil.x, pupil.y, pupil.width, pupil.height);
      }
    }
    return pupil;
  }
  public Rectangle findGlint(Rectangle _pupil) {
    // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL
    int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2);
    int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2);
    int glintTop = _pupil.y;
    int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);
    ArrayList glintsCandidates = new ArrayList();
    for (int row = glintTop; row < glintBottom; row++) {
      for (int col = glintEdgeL; col < glintEdgeR; col++) {
        int offset = row * video.width + col;
        int thisPixel = video.pixels[offset];
        //look for bright things
        if (brightness(thisPixel) > glintThreshold) {
          if (debug) video.pixels[offset] = 0;
          // be pessimistic
          boolean foundAHome = false;
          // look throught the existing
          for (int i = 0; i < glintsCandidates.size(); i++) {
            Rectangle existingBox = (Rectangle) glintsCandidates.get (i);
            // is this spot in an existing box
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
            inflatedBox.grow(reach, reach); // widen it's reach
            if (inflatedBox.contains(col, row)) {
              existingBox.add(col, row);
              foundAHome = true; // no need to make a new one
              break; // no need to look through the rest of the boxes
            }
          }
          // if this does not belong to one of the existing boxes make a new one at this place
          if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0));
        }
      }
    }
    // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL
    Rectangle glint = findClosestMostBigOne(glintsCandidates,

_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0);

    stroke(0, 0, 255);
    if (debug) {// show all the candidate
      // show the edges of the search for the glint
      stroke(0, 75, 200);
      rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop);
      for (int i = 0; i < glintsCandidates.size(); i++) {
        Rectangle thisBox = (Rectangle) glintsCandidates.get(i);
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
      } // show the winner in red
      if (glint != null) {
        stroke(255, 0, 0);
        rect(glint.x, glint.y, glint.width, glint.height);
      }
    }
    return glint;
  }
  public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) {
    // check every combination of shapes for overlap
    // make the repeat loop backwards so you delete off the bottom of the stack
    for (int i = _shapes.size() - 1; i > -1; i--) {
      // only check the ones up
      Rectangle shape1 = (Rectangle) _shapes.get(i);
      Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box
      inflatedShape1.grow(_consolidateReachX,

_consolidateReachY); // widen it's reach

      for (int j = i - 1; j > -1; j--) {
        Rectangle shape2 = (Rectangle) _shapes.get(j);
        if (inflatedShape1.intersects(shape2)) {
          shape1.add(shape2);
          // System.out.println("Remove" + j);
          _shapes.remove(j);
          break;
        }
      }
    }
  }
  public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) {
    if (_allRects.size() == 0) return null;
    int winner = 0;
    float closest = 1000;
    for (int i = 0; i < _allRects.size(); i++) {
      Rectangle thisRect = (Rectangle)  _allRects.get(i);
      if (thisRect.width * thisRect.height < _minArea) continue;
      float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2);
      if (thisDist < closest) {
        closest = thisDist;
        winner = i;
      }
    }
    return (Rectangle) _allRects.get(winner);
  }
  void stop() {
  }

}