82
edits
Changes
→Visualization
==Visualization==
*Functional Diagram
program code: import java.util.ArrayList; import processing.core.PApplet;import processing.video.Capture; public class ICU extends PApplet { final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3; final int KIM = 4; int item = JESSICA;boolean usingMouse = false; //to use mouse for preso, if needed! Capture video;// regular processing libary int pupilThreshold = 100; int glintThreshold = 240; int edge = 50; int reach = 5; int minPupilArea = 75; long elapsedTime; float calibrationSlope; int calibrationIntercept; Rectangle pupil; Rectangle glint; CoordinatesTranslator calibrator; PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; long lastVideo = 0; //variables for easingfloat crosshairLocationOnScreenX;float crosshairLocationOnScreenY;float targetX, targetY;float easing = 0.05; boolean debug = true; PImage testImage; boolean leaveTrails; static public void main(String _args[]) { PApplet.main(new String[] { "tracking.Eyetrack" } ); }int num = 20;// Images must be in the "data" directory to load correctly float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius; public void setup() { size(600, 800); noStroke(); smooth(); //ellipseMode(RADIUS); //rectMode(CORNERS); println( Capture.list()); video = new Capture(this, width, height, Capture.list()[Image:vis145bpic4] ); //video.settings(); testImage = loadImage("mm.jpg"); size(testImage.width,testImage.height); calibrator = new CoordinatesTranslator(); image(testImage,0,0); smooth(); img = loadImage("korean-girls-kang-su-yeon.jpg"); imgJessica = loadImage("Jessica_alba.jpg"); imgMaggie = loadImage("maggie_q_1.jpg"); imgHyori = loadImage("lee_hyori.jpg"); imgNamie = loadImage("namie_amuro_1.jpg"); noCursor(); imageMode(CENTER); } public void draw() { background(225); drawTarget(); stroke(225,0,0); noFill(); //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX; if (abs(mouseX - mx) > 0.1) { mx = mx + (mouseX - mx) * easing; } if (abs(mouseY - my) > 0.1) { my = my + (mouseY- my) * easing; } mx = constrain(mx, inner, width - inner); my = constrain(my, inner, height - inner); //fill(76); //rect(edge, edge, width-edge, height-edge); fill(0); ellipse(mx, my, radius, radius); int which = frameCount % num; mx[which] = mouseX; my[which] = mouseY; for (int i = 0; i < num; i++) { //which+1 is the smallest (the oldest in the array) int index = (which+1 + i) % num; ellipse(mx[index], my[index], i, i); } if(abs(dx) > 1) { crosshairLocationOnScreenX += dx * easing; } //targetY = drawY; float dy = drawY - crosshairLocationOnScreenY; if(abs(dy) > 1) { crosshairLocationOnScreenY += dy * easing; } if (video.available() && (millis() > lastVideo + 10)) { lastVideo = millis(); video.read(); long startTime = millis(); elapsedTime = millis() - startTime; if (debug) { background(0); image(video, 0, 0); } else{ if (leaveTrails == false) image(testImage,0,0); if (calibrating) { image(testImage,0,0); } if (pupil != null) { ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40); //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width)); //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/ //drawX = adjustedPoint[0]; //drawY = adjustedPoint[1];*/ //println(adjustedPoint[0]+", "+adjustedPoint[1]); } } pupil = findPupil(); glint = null; if (pupil != null) glint = findGlint(pupil); //if the glint and the pupil are present if (glint != null && pupil != null) { // if we got both // find out where the pupil is relative to the glint int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2; int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2 ; //if the system is calibrating if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at //draw the dots for the user to look at for calibration purposes placeToDrawTarget = placeToLook; if (placeToDrawTarget.x > 0 &&calibrator.isCurrentlyCalibrating()) { stroke(0); fill(0); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look stroke(255); fill(255); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look stroke(40,40,255); fill(40,40,255); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look stroke(255,0,0); fill(255,0,0); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look stroke(255,255,0); fill(255,255,0); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look } //CHANGE THIS TO ELMO calibrator.doCalibrationRoutine(true,rawX, rawY); println("calibrating"); } //if the system is not calibrating, else { //use the calibrator to find out where the x,y from camera corolate to on image int[] adjustedPoint = calibrator.translate(rawX, rawY); println("Adjusted x" + adjustedPoint[0] + " adjusted y" + adjustedPoint[1]); //if the mouse is not being used, draw the crosshairs at adjusted points if(!usingMouse) { drawX = adjustedPoint[0]; drawY = adjustedPoint[1]; } } } } } public Rectangle findPupil() { // /FIND THE PUPIL ArrayList boxes = new ArrayList(); for (int row = edge; row < video.height - edge * 2; row++) { for (int col = edge; col < video.width - edge * 2; col++) { int offset = row * video.width + col; int thisPixel = video.pixels[offset]; //look for dark things if (brightness(thisPixel) < pupilThreshold) { video.pixels[offset] = 0; // be pessimistic boolean foundAHome = false; // look throught the existing for (int i = 0; i < boxes.size(); i++) { Rectangle existingBox = (Rectangle) boxes.get(i); // is this spot in an existing box Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box inflatedBox.grow(reach, reach); // widen it's reach if (inflatedBox.contains(col, row)) { existingBox.add(col, row); foundAHome = true; // no need to make a new one break; // no need to look through the rest of the boxes } } // if this does not belong to one of the existing boxes make a new one at this place if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0)); } } } consolidate(boxes, 0, 0); // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea); if (debug) { // show the the edges of the search fill(0, 0, 0, 0); stroke(0, 255, 0); rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge); // show all the pupil candidates stroke(0, 0, 0); for (int i = 0; i < boxes.size(); i++) { Rectangle thisBox = (Rectangle) boxes.get(i); rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height); } // show the winning pupil candidate in red if (pupil != null) { stroke(255, 0, 0); rect(pupil.x, pupil.y, pupil.width, pupil.height); } } return pupil; } public Rectangle findGlint(Rectangle _pupil) { // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2); int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2); int glintTop = _pupil.y; int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2); ArrayList glintsCandidates = new ArrayList(); for (int row = glintTop; row < glintBottom; row++) { for (int col = glintEdgeL; col < glintEdgeR; col++) { int offset = row * video.width + col; int thisPixel = video.png|none|300pxpixels[offset]; //look for bright things if (brightness(thisPixel) > glintThreshold) { if (debug) video.pixels[offset]= 0; // be pessimistic boolean foundAHome = false; // look throught the existing for (int i = 0; i < glintsCandidates.size(); i++) { Rectangle existingBox = (Rectangle) glintsCandidates.get (i); // is this spot in an existing box Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box inflatedBox.grow(reach, reach); // widen it's reach if (inflatedBox.contains(col, row)) { existingBox.add(col, row); foundAHome = true; // no need to make a new one break; // no need to look through the rest of the boxes } } // if this does not belong to one of the existing boxes make a new one at this place if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0)); } } } // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL Rectangle glint = findClosestMostBigOne(glintsCandidates,_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0); stroke(0, 0, 255); if (debug) {// show all the candidate // show the edges of the search for the glint stroke(0, 75, 200); rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop); for (int i = 0; i < glintsCandidates.size(); i++) { Rectangle thisBox = (Rectangle) glintsCandidates.get(i); rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height); } // show the winner in red if (glint != null) { stroke(255, 0, 0); rect(glint.x, glint.y, glint.width, glint.height); } } return glint; } public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) { // check every combination of shapes for overlap // make the repeat loop backwards so you delete off the bottom of the stack for (int i = _shapes.size() - 1; i > -1; i--) { // only check the ones up Rectangle shape1 = (Rectangle) _shapes.get(i); Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box inflatedShape1.grow(_consolidateReachX,_consolidateReachY); // widen it's reach for (int j = i - 1; j > -1; j--) { Rectangle shape2 = (Rectangle) _shapes.get(j); if (inflatedShape1.intersects(shape2)) { shape1.add(shape2); // System.out.println("Remove" + j); _shapes.remove(j); break; } } } } public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) { if (_allRects.size() == 0) return null; int winner = 0; float closest = 1000; for (int i = 0; i < _allRects.size(); i++) { Rectangle thisRect = (Rectangle) _allRects.get(i); if (thisRect.width * thisRect.height < _minArea) continue; float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2); if (thisDist < closest) { closest = thisDist; winner = i; } } return (Rectangle) _allRects.get(winner); } void stop() { }}
