Difference between revisions of "Wii - remote composer - Javi Lee"

From Robert-Depot
Jump to: navigation, search
(Visualization)
(Visualization)
 
(2 intermediate revisions by the same user not shown)
Line 2: Line 2:
 
*Motivation
 
*Motivation
  
I was just curious to see if one could compose music with the Wii remote. I've been very influenced lately by many friend who create their own music and so I wanted to have a "musical" aspect in this project. My basic idea would be set up a screen that will be cut up into squares, (not visibly) and these squares will be holding certain notes. With the Wii remote, one could go over these squares and press a button on the remote to 'activate' the sound. The program will automatically save this input and have the possibility to play back the music after wards.  
+
I wanted to explore the difference between the minds of men and women. I wanted to see what the males prefer over the preferences of the female and vice versa. In a way to explore the psychological development of the human brain and if gender has a factor to it.  
  
 
*Interactive paradigm
 
*Interactive paradigm
  
I wanted to take the physical space and turn it into "concert" hall. The user will be able to compose 'music' without having to go into a secluded room to use technologically advanced computers and programs to compose a small piece of music. This will be a real-time composition of music as if he/she were to be a composer for a band at a concert.
+
I wanted to take the physical space and turn it into experiment lab. The user will not be able to see that the program is tracking what he/she is looking at but the observers will be able to see.  
  
 
*Technical Description
 
*Technical Description
  
I want to use Processing to be able to connect the the sensor and the accelerometer in the Wii remote. The processing program will split the screen into grids where each square of the grid will hold a pitch/sound. Then the user will be able to use the sensor of the Wii remote to track where the remote will be pointing at the screen. The accelerometer will help to move around the screen as well. Then by pressing a button, the user could play the sound and 'save' the sound so after the user presses so many buttons, he/she will be able to replay the sequence of sounds saved.
+
I want to use Processing to display the tracks of where the eyeball will be looking at. It will display pictures and the participant will just look at the pictures. I wanted to put an eyeball tracker that will track where the eye is looking at. In this way it will track what the participants are looking at.
  
 
==Visualization==
 
==Visualization==
 
*Functional Diagram
 
*Functional Diagram
[[Image:vis145bpic.png|none|300px]]
+
program code:
 +
 
 +
import java.util.ArrayList;
 +
 
 +
import processing.core.PApplet;
 +
import processing.video.Capture;
 +
 
 +
public class ICU extends PApplet {
 +
  final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3;
 +
  final int KIM = 4;
 +
  int item = JESSICA;
 +
boolean usingMouse = false; //to use mouse for preso, if needed!
 +
 
 +
  Capture video;// regular processing libary
 +
  int pupilThreshold = 100;
 +
  int glintThreshold = 240;
 +
  int edge = 50;
 +
  int reach = 5;
 +
  int minPupilArea = 75;
 +
  long elapsedTime;
 +
  float calibrationSlope;
 +
  int calibrationIntercept;
 +
  Rectangle pupil;
 +
  Rectangle glint;
 +
  CoordinatesTranslator calibrator;
 +
 
 +
PImage img, imgJessica, imgMaggie, imgHyori, imgNamie;
 +
long lastVideo = 0;
 +
 +
//variables for easing
 +
float crosshairLocationOnScreenX;
 +
float crosshairLocationOnScreenY;
 +
float targetX, targetY;
 +
float easing = 0.05;
 +
  boolean debug = true;
 +
  PImage testImage;
 +
  boolean leaveTrails;
 +
  static public void main(String _args[]) {
 +
    PApplet.main(new String[] {
 +
      "tracking.Eyetrack"                  }
 +
    );
 +
  }
 +
int num = 20;
 +
// Images must be in the "data" directory to load correctly
 +
float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius;
 +
 
 +
public void setup() {
 +
  size(600, 800);
 +
  noStroke();
 +
  smooth();
 +
  //ellipseMode(RADIUS);
 +
  //rectMode(CORNERS);
 +
  println( Capture.list());
 +
    video = new Capture(this, width, height, Capture.list()[4] );
 +
    //video.settings();
 +
    testImage = loadImage("mm.jpg");
 +
    size(testImage.width,testImage.height);
 +
    calibrator = new CoordinatesTranslator();
 +
    image(testImage,0,0);
 +
 
 +
      smooth();
 +
     
 +
      img = loadImage("korean-girls-kang-su-yeon.jpg");
 +
      imgJessica = loadImage("Jessica_alba.jpg");
 +
      imgMaggie = loadImage("maggie_q_1.jpg");
 +
      imgHyori = loadImage("lee_hyori.jpg");
 +
      imgNamie = loadImage("namie_amuro_1.jpg");
 +
 
 +
     
 +
 
 +
    noCursor();
 +
 
 +
imageMode(CENTER);
 +
 
 +
}
 +
 
 +
public void draw() {
 +
  background(225);
 +
  drawTarget();
 +
  stroke(225,0,0);
 +
  noFill();
 +
  //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX;
 +
    if (abs(mouseX - mx) > 0.1) {
 +
    mx = mx + (mouseX - mx) * easing;
 +
  }
 +
  if (abs(mouseY - my) > 0.1) {
 +
    my = my + (mouseY- my) * easing;
 +
  }
 +
 
 +
  mx = constrain(mx, inner, width - inner);
 +
  my = constrain(my, inner, height - inner);
 +
  //fill(76);
 +
  //rect(edge, edge, width-edge, height-edge);
 +
  fill(0);
 +
  ellipse(mx, my, radius, radius);
 +
 
 +
  int which = frameCount % num;
 +
  mx[which] = mouseX;
 +
  my[which] = mouseY;
 +
 
 +
  for (int i = 0; i < num; i++) {
 +
      //which+1 is the smallest (the oldest in the array)
 +
    int index = (which+1 + i) % num;
 +
    ellipse(mx[index], my[index], i, i); }
 +
  if(abs(dx) > 1) {
 +
    crosshairLocationOnScreenX += dx * easing;
 +
  }
 +
 
 +
  //targetY = drawY;
 +
  float dy = drawY - crosshairLocationOnScreenY;
 +
  if(abs(dy) > 1) {
 +
    crosshairLocationOnScreenY += dy * easing;
 +
  }
 +
 
 +
if (video.available() && (millis() > lastVideo + 10)) {
 +
  lastVideo = millis();
 +
      video.read();
 +
      long startTime = millis();
 +
      elapsedTime = millis() - startTime;
 +
      if (debug) {
 +
        background(0);
 +
        image(video, 0, 0);
 +
      }
 +
      else{
 +
        if (leaveTrails == false) image(testImage,0,0);
 +
          if (calibrating) { image(testImage,0,0); }
 +
          if (pupil != null) {
 +
            ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40);
 +
            //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width));
 +
            //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/
 +
            //drawX = adjustedPoint[0];
 +
            //drawY = adjustedPoint[1];*/
 +
            //println(adjustedPoint[0]+", "+adjustedPoint[1]);
 +
          }
 +
      }
 +
      pupil = findPupil();
 +
      glint = null;
 +
      if (pupil != null) glint = findGlint(pupil);
 +
      //if the glint and the pupil are present
 +
      if (glint != null && pupil != null) { // if we got both
 +
        // find out where the pupil is relative to the glint
 +
        int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2;
 +
        int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2  ;
 +
        //if the system is calibrating
 +
        if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode
 +
          Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at
 +
          //draw the dots for the user to look at for calibration purposes
 +
          placeToDrawTarget = placeToLook;
 +
          if (placeToDrawTarget.x > 0 &&
 +
calibrator.isCurrentlyCalibrating()) {
 +
          stroke(0);
 +
          fill(0);
 +
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look
 +
          stroke(255);
 +
          fill(255);
 +
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look
 +
          stroke(40,40,255);
 +
          fill(40,40,255);
 +
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look
 +
          stroke(255,0,0);
 +
          fill(255,0,0);
 +
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look
 +
          stroke(255,255,0);
 +
          fill(255,255,0);
 +
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look
 +
          }
 +
          //CHANGE THIS TO ELMO
 +
 
 +
          calibrator.doCalibrationRoutine(true,rawX, rawY);
 +
          println("calibrating");
 +
 
 +
        }
 +
        //if the system is not calibrating,
 +
        else {
 +
          //use the calibrator to find out where the x,y from camera corolate to on image
 +
          int[] adjustedPoint = calibrator.translate(rawX, rawY);
 +
          println("Adjusted x" + adjustedPoint[0]  + " adjusted y" + adjustedPoint[1]);
 +
 
 +
          //if the mouse is not being used, draw the crosshairs at adjusted points
 +
          if(!usingMouse) {
 +
            drawX = adjustedPoint[0];
 +
            drawY = adjustedPoint[1];
 +
          }
 +
        }
 +
 
 +
      }
 +
 
 +
    }
 +
 
 +
    }
 +
 
 +
  public Rectangle findPupil() {
 +
    // /FIND THE PUPIL
 +
    ArrayList boxes = new ArrayList();
 +
    for (int row = edge; row < video.height - edge * 2; row++) {
 +
      for (int col = edge; col < video.width - edge * 2; col++) {
 +
        int offset = row * video.width + col;
 +
        int thisPixel = video.pixels[offset];
 +
        //look for dark things
 +
        if (brightness(thisPixel) < pupilThreshold) {
 +
          video.pixels[offset] = 0;
 +
          // be pessimistic
 +
          boolean foundAHome = false;
 +
          // look throught the existing
 +
          for (int i = 0; i < boxes.size(); i++) {
 +
            Rectangle existingBox = (Rectangle) boxes.get(i);
 +
            // is this spot in an existing box
 +
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
 +
            inflatedBox.grow(reach, reach); // widen it's reach
 +
            if (inflatedBox.contains(col, row)) {
 +
              existingBox.add(col, row);
 +
              foundAHome = true; // no need to make a new one
 +
              break; // no need to look through the rest of the boxes
 +
            }
 +
          }
 +
          // if this does not belong to one of the existing boxes make a new one at this place
 +
          if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0));
 +
        }
 +
      }
 +
    }
 +
 
 +
    consolidate(boxes, 0, 0);
 +
 
 +
    // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER
 +
    Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea);
 +
    if (debug) {
 +
      // show the the edges of the search
 +
      fill(0, 0, 0, 0);
 +
      stroke(0, 255, 0);
 +
      rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge);
 +
      // show all the pupil candidates
 +
      stroke(0, 0, 0);
 +
      for (int i = 0; i < boxes.size(); i++) {
 +
        Rectangle thisBox = (Rectangle) boxes.get(i);
 +
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
 +
      }
 +
      // show the winning pupil candidate in red
 +
      if (pupil != null) {
 +
        stroke(255, 0, 0);
 +
        rect(pupil.x, pupil.y, pupil.width, pupil.height);
 +
      }
 +
    }
 +
    return pupil;
 +
  }
 +
 
 +
  public Rectangle findGlint(Rectangle _pupil) {
 +
    // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL
 +
    int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2);
 +
    int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2);
 +
    int glintTop = _pupil.y;
 +
    int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);
 +
 
 +
    ArrayList glintsCandidates = new ArrayList();
 +
    for (int row = glintTop; row < glintBottom; row++) {
 +
      for (int col = glintEdgeL; col < glintEdgeR; col++) {
 +
        int offset = row * video.width + col;
 +
        int thisPixel = video.pixels[offset];
 +
        //look for bright things
 +
        if (brightness(thisPixel) > glintThreshold) {
 +
          if (debug) video.pixels[offset] = 0;
 +
          // be pessimistic
 +
          boolean foundAHome = false;
 +
          // look throught the existing
 +
          for (int i = 0; i < glintsCandidates.size(); i++) {
 +
            Rectangle existingBox = (Rectangle) glintsCandidates.get (i);
 +
            // is this spot in an existing box
 +
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
 +
            inflatedBox.grow(reach, reach); // widen it's reach
 +
            if (inflatedBox.contains(col, row)) {
 +
              existingBox.add(col, row);
 +
              foundAHome = true; // no need to make a new one
 +
              break; // no need to look through the rest of the boxes
 +
            }
 +
          }
 +
          // if this does not belong to one of the existing boxes make a new one at this place
 +
          if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0));
 +
        }
 +
      }
 +
    }
 +
    // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL
 +
    Rectangle glint = findClosestMostBigOne(glintsCandidates,
 +
_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0);
 +
    stroke(0, 0, 255);
 +
 
 +
    if (debug) {// show all the candidate
 +
      // show the edges of the search for the glint
 +
      stroke(0, 75, 200);
 +
      rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop);
 +
      for (int i = 0; i < glintsCandidates.size(); i++) {
 +
        Rectangle thisBox = (Rectangle) glintsCandidates.get(i);
 +
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
 +
      } // show the winner in red
 +
      if (glint != null) {
 +
        stroke(255, 0, 0);
 +
        rect(glint.x, glint.y, glint.width, glint.height);
 +
      }
 +
    }
 +
 
 +
    return glint;
 +
  }
 +
 
 +
  public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) {
 +
    // check every combination of shapes for overlap
 +
    // make the repeat loop backwards so you delete off the bottom of the stack
 +
    for (int i = _shapes.size() - 1; i > -1; i--) {
 +
      // only check the ones up
 +
      Rectangle shape1 = (Rectangle) _shapes.get(i);
 +
      Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box
 +
      inflatedShape1.grow(_consolidateReachX,
 +
_consolidateReachY); // widen it's reach
 +
 
 +
      for (int j = i - 1; j > -1; j--) {
 +
        Rectangle shape2 = (Rectangle) _shapes.get(j);
 +
        if (inflatedShape1.intersects(shape2)) {
 +
          shape1.add(shape2);
 +
          // System.out.println("Remove" + j);
 +
          _shapes.remove(j);
 +
          break;
 +
        }
 +
      }
 +
    }
 +
  }
 +
 
 +
  public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) {
 +
    if (_allRects.size() == 0) return null;
 +
    int winner = 0;
 +
    float closest = 1000;
 +
 
 +
    for (int i = 0; i < _allRects.size(); i++) {
 +
      Rectangle thisRect = (Rectangle)  _allRects.get(i);
 +
      if (thisRect.width * thisRect.height < _minArea) continue;
 +
      float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2);
 +
      if (thisDist < closest) {
 +
        closest = thisDist;
 +
        winner = i;
 +
      }
 +
    }
 +
    return (Rectangle) _allRects.get(winner);
 +
  }
 +
  void stop() {
 +
  }
 +
}

Latest revision as of 14:23, 26 May 2010

Description

  • Motivation

I wanted to explore the difference between the minds of men and women. I wanted to see what the males prefer over the preferences of the female and vice versa. In a way to explore the psychological development of the human brain and if gender has a factor to it.

  • Interactive paradigm

I wanted to take the physical space and turn it into experiment lab. The user will not be able to see that the program is tracking what he/she is looking at but the observers will be able to see.

  • Technical Description

I want to use Processing to display the tracks of where the eyeball will be looking at. It will display pictures and the participant will just look at the pictures. I wanted to put an eyeball tracker that will track where the eye is looking at. In this way it will track what the participants are looking at.

Visualization

  • Functional Diagram

program code:

import java.util.ArrayList;

import processing.core.PApplet; import processing.video.Capture;

public class ICU extends PApplet {

 final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3;
  final int KIM = 4;
  int item = JESSICA;

boolean usingMouse = false; //to use mouse for preso, if needed!

  Capture video;// regular processing libary
  int pupilThreshold = 100;
  int glintThreshold = 240;
  int edge = 50;
  int reach = 5;
  int minPupilArea = 75;
  long elapsedTime;
  float calibrationSlope;
  int calibrationIntercept;
  Rectangle pupil;
  Rectangle glint;
  CoordinatesTranslator calibrator;

PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; long lastVideo = 0;

//variables for easing float crosshairLocationOnScreenX; float crosshairLocationOnScreenY; float targetX, targetY; float easing = 0.05;

  boolean debug = true;
  PImage testImage;
  boolean leaveTrails;
  static public void main(String _args[]) {
    PApplet.main(new String[] {
      "tracking.Eyetrack"                  }
   );
  }

int num = 20; // Images must be in the "data" directory to load correctly float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius;

public void setup() {

  size(600, 800);
  noStroke();
  smooth();
  //ellipseMode(RADIUS);
  //rectMode(CORNERS);
  println( Capture.list());
   video = new Capture(this, width, height, Capture.list()[4] );
    //video.settings();
    testImage = loadImage("mm.jpg");
    size(testImage.width,testImage.height);
    calibrator = new CoordinatesTranslator();
    image(testImage,0,0);
      smooth();
      
      img = loadImage("korean-girls-kang-su-yeon.jpg");
      imgJessica = loadImage("Jessica_alba.jpg");
      imgMaggie = loadImage("maggie_q_1.jpg");
      imgHyori = loadImage("lee_hyori.jpg");
      imgNamie = loadImage("namie_amuro_1.jpg");


    noCursor();

imageMode(CENTER);

}

public void draw() {

  background(225);
  drawTarget();
  stroke(225,0,0);
  noFill();
  //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX;
    if (abs(mouseX - mx) > 0.1) {
    mx = mx + (mouseX - mx) * easing;
  }
  if (abs(mouseY - my) > 0.1) {
    my = my + (mouseY- my) * easing;
  }
  mx = constrain(mx, inner, width - inner);
  my = constrain(my, inner, height - inner);
  //fill(76);
  //rect(edge, edge, width-edge, height-edge);
  fill(0);
  ellipse(mx, my, radius, radius);
  int which = frameCount % num;
  mx[which] = mouseX;
  my[which] = mouseY;
  for (int i = 0; i < num; i++) {
     //which+1 is the smallest (the oldest in the array)
    int index = (which+1 + i) % num;
    ellipse(mx[index], my[index], i, i); }
  if(abs(dx) > 1) {
    crosshairLocationOnScreenX += dx * easing;
  }
  //targetY = drawY;
  float dy = drawY - crosshairLocationOnScreenY;
  if(abs(dy) > 1) {
    crosshairLocationOnScreenY += dy * easing;
  }
  

if (video.available() && (millis() > lastVideo + 10)) {

  lastVideo = millis();
      video.read();
      long startTime = millis();
      elapsedTime = millis() - startTime;
      if (debug) {
        background(0);
        image(video, 0, 0);
      }
      else{
        if (leaveTrails == false) image(testImage,0,0);
         if (calibrating) { image(testImage,0,0); }
         if (pupil != null) {
           ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40);
           //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width));
           //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/
           //drawX = adjustedPoint[0];
           //drawY = adjustedPoint[1];*/
           //println(adjustedPoint[0]+", "+adjustedPoint[1]);
         }
      }
      pupil = findPupil();
      glint = null;
      if (pupil != null) glint = findGlint(pupil);
      //if the glint and the pupil are present
      if (glint != null && pupil != null) { // if we got both
        // find out where the pupil is relative to the glint
        int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2;
        int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2  ;
        //if the system is calibrating
        if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode
          Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at
          //draw the dots for the user to look at for calibration purposes
          placeToDrawTarget = placeToLook;
          if (placeToDrawTarget.x > 0 &&

calibrator.isCurrentlyCalibrating()) {

          stroke(0);
          fill(0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look
          stroke(255);
          fill(255);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look
          stroke(40,40,255);
          fill(40,40,255);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look
          stroke(255,0,0);
          fill(255,0,0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look
          stroke(255,255,0);
          fill(255,255,0);
          ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look
          }
          //CHANGE THIS TO ELMO
          calibrator.doCalibrationRoutine(true,rawX, rawY);
          println("calibrating");
        }
        //if the system is not calibrating,
        else {
          //use the calibrator to find out where the x,y from camera corolate to on image
          int[] adjustedPoint = calibrator.translate(rawX, rawY);
          println("Adjusted x" + adjustedPoint[0]  + " adjusted y" + adjustedPoint[1]);
          //if the mouse is not being used, draw the crosshairs at adjusted points
          if(!usingMouse) {
            drawX = adjustedPoint[0];
            drawY = adjustedPoint[1];
          }
        }
      }
    }
    }
  public Rectangle findPupil() {
    // /FIND THE PUPIL
    ArrayList boxes = new ArrayList();
    for (int row = edge; row < video.height - edge * 2; row++) {
      for (int col = edge; col < video.width - edge * 2; col++) {
        int offset = row * video.width + col;
        int thisPixel = video.pixels[offset];
        //look for dark things
        if (brightness(thisPixel) < pupilThreshold) {
          video.pixels[offset] = 0;
          // be pessimistic
          boolean foundAHome = false;
          // look throught the existing
          for (int i = 0; i < boxes.size(); i++) {
            Rectangle existingBox = (Rectangle) boxes.get(i);
            // is this spot in an existing box
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
            inflatedBox.grow(reach, reach); // widen it's reach
            if (inflatedBox.contains(col, row)) {
              existingBox.add(col, row);
              foundAHome = true; // no need to make a new one
              break; // no need to look through the rest of the boxes
            }
          }
          // if this does not belong to one of the existing boxes make a new one at this place
          if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0));
        }
      }
    }
    consolidate(boxes, 0, 0);
    // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER
    Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea);
    if (debug) {
      // show the the edges of the search
      fill(0, 0, 0, 0);
      stroke(0, 255, 0);
      rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge);
      // show all the pupil candidates
      stroke(0, 0, 0);
      for (int i = 0; i < boxes.size(); i++) {
        Rectangle thisBox = (Rectangle) boxes.get(i);
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
      }
      // show the winning pupil candidate in red
      if (pupil != null) {
        stroke(255, 0, 0);
        rect(pupil.x, pupil.y, pupil.width, pupil.height);
      }
    }
    return pupil;
  }
  public Rectangle findGlint(Rectangle _pupil) {
    // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL
    int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2);
    int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2);
    int glintTop = _pupil.y;
    int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);
    ArrayList glintsCandidates = new ArrayList();
    for (int row = glintTop; row < glintBottom; row++) {
      for (int col = glintEdgeL; col < glintEdgeR; col++) {
        int offset = row * video.width + col;
        int thisPixel = video.pixels[offset];
        //look for bright things
        if (brightness(thisPixel) > glintThreshold) {
          if (debug) video.pixels[offset] = 0;
          // be pessimistic
          boolean foundAHome = false;
          // look throught the existing
          for (int i = 0; i < glintsCandidates.size(); i++) {
            Rectangle existingBox = (Rectangle) glintsCandidates.get (i);
            // is this spot in an existing box
            Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box
            inflatedBox.grow(reach, reach); // widen it's reach
            if (inflatedBox.contains(col, row)) {
              existingBox.add(col, row);
              foundAHome = true; // no need to make a new one
              break; // no need to look through the rest of the boxes
            }
          }
          // if this does not belong to one of the existing boxes make a new one at this place
          if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0));
        }
      }
    }
    // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL
    Rectangle glint = findClosestMostBigOne(glintsCandidates,

_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0);

    stroke(0, 0, 255);
    if (debug) {// show all the candidate
      // show the edges of the search for the glint
      stroke(0, 75, 200);
      rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop);
      for (int i = 0; i < glintsCandidates.size(); i++) {
        Rectangle thisBox = (Rectangle) glintsCandidates.get(i);
        rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height);
      } // show the winner in red
      if (glint != null) {
        stroke(255, 0, 0);
        rect(glint.x, glint.y, glint.width, glint.height);
      }
    }
    return glint;
  }
  public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) {
    // check every combination of shapes for overlap
    // make the repeat loop backwards so you delete off the bottom of the stack
    for (int i = _shapes.size() - 1; i > -1; i--) {
      // only check the ones up
      Rectangle shape1 = (Rectangle) _shapes.get(i);
      Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box
      inflatedShape1.grow(_consolidateReachX,

_consolidateReachY); // widen it's reach

      for (int j = i - 1; j > -1; j--) {
        Rectangle shape2 = (Rectangle) _shapes.get(j);
        if (inflatedShape1.intersects(shape2)) {
          shape1.add(shape2);
          // System.out.println("Remove" + j);
          _shapes.remove(j);
          break;
        }
      }
    }
  }
  public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) {
    if (_allRects.size() == 0) return null;
    int winner = 0;
    float closest = 1000;
    for (int i = 0; i < _allRects.size(); i++) {
      Rectangle thisRect = (Rectangle)  _allRects.get(i);
      if (thisRect.width * thisRect.height < _minArea) continue;
      float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2);
      if (thisDist < closest) {
        closest = thisDist;
        winner = i;
      }
    }
    return (Rectangle) _allRects.get(winner);
  }
  void stop() {
  }

}