Changes

I was just curious wanted to see if one could compose music with explore the difference between the Wii remoteminds of men and women. I've been very influenced lately by many friend who create their own music and so I wanted to have a "musical" aspect in this project. My basic idea would be set up a screen that will be cut up into squares, (not visibly) and these squares will be holding certain notes. With see what the Wii remote, one could go males prefer over these squares the preferences of the female and press vice versa. In a button on way to explore the remote to 'activate' psychological development of the sound. The program will automatically save this input human brain and have the possibility if gender has a factor to play back the music after wardsit.

I wanted to take the physical space and turn it into "concert" hallexperiment lab. The user will not be able to compose 'music' without having to go into a secluded room to use technologically advanced computers and programs to compose a small piece of music. This will be a real-time composition of music as if see that the program is tracking what he/she were is looking at but the observers will be able to be a composer for a band at a concertsee.

I want to use Processing to be able to connect display the the sensor and the accelerometer in the Wii remote. The processing program will split the screen into grids tracks of where each square of the grid eyeball will hold a pitch/soundbe looking at. Then It will display pictures and the user participant will be able to use just look at the sensor of the Wii remote pictures. I wanted to put an eyeball tracker that will track where the remote will be pointing eye is looking at the screen. The accelerometer In this way it will help to move around track what the screen as well. Then by pressing a button, the user could play the sound and 'save' the sound so after the user presses so many buttons, he/she will be able to replay the sequence of sounds savedparticipants are looking at.

program code: import java.util.ArrayList; import processing.core.PApplet;import processing.video.Capture; public class ICU extends PApplet { final int JESSICA = 0, MAGGIE = 1, HYORI = 2, NAMIE = 3; final int KIM = 4; int item = JESSICA;boolean usingMouse = false; //to use mouse for preso, if needed!  Capture video;// regular processing libary int pupilThreshold = 100; int glintThreshold = 240; int edge = 50; int reach = 5; int minPupilArea = 75; long elapsedTime; float calibrationSlope; int calibrationIntercept; Rectangle pupil; Rectangle glint; CoordinatesTranslator calibrator; PImage img, imgJessica, imgMaggie, imgHyori, imgNamie; long lastVideo = 0; //variables for easingfloat crosshairLocationOnScreenX;float crosshairLocationOnScreenY;float targetX, targetY;float easing = 0.05; boolean debug = true; PImage testImage; boolean leaveTrails; static public void main(String _args[]) { PApplet.main(new String[] { "tracking.Eyetrack" } ); }int num = 20;// Images must be in the "data" directory to load correctly float mx[] = new float[num]; float my[] = new float[num]; float easing = 0.05; int radius = 8; int edge = 0; int inner = edge + radius; public void setup() { size(600, 800); noStroke(); smooth(); //ellipseMode(RADIUS); //rectMode(CORNERS); println( Capture.list()); video = new Capture(this, width, height, Capture.list()[Image:vis145bpic4] ); //video.settings(); testImage = loadImage("mm.jpg"); size(testImage.width,testImage.height); calibrator = new CoordinatesTranslator(); image(testImage,0,0);  smooth(); img = loadImage("korean-girls-kang-su-yeon.jpg"); imgJessica = loadImage("Jessica_alba.jpg"); imgMaggie = loadImage("maggie_q_1.jpg"); imgHyori = loadImage("lee_hyori.jpg"); imgNamie = loadImage("namie_amuro_1.jpg");    noCursor(); imageMode(CENTER); } public void draw() { background(225); drawTarget(); stroke(225,0,0); noFill(); //draw crosshairs on screen, and add easing for smoother transitions float dx = drawX - crosshairLocationOnScreenX; if (abs(mouseX - mx) > 0.1) { mx = mx + (mouseX - mx) * easing; } if (abs(mouseY - my) > 0.1) { my = my + (mouseY- my) * easing; }  mx = constrain(mx, inner, width - inner); my = constrain(my, inner, height - inner); //fill(76); //rect(edge, edge, width-edge, height-edge); fill(0); ellipse(mx, my, radius, radius);  int which = frameCount % num; mx[which] = mouseX; my[which] = mouseY;  for (int i = 0; i < num; i++) { //which+1 is the smallest (the oldest in the array) int index = (which+1 + i) % num; ellipse(mx[index], my[index], i, i); } if(abs(dx) > 1) { crosshairLocationOnScreenX += dx * easing; }  //targetY = drawY; float dy = drawY - crosshairLocationOnScreenY; if(abs(dy) > 1) { crosshairLocationOnScreenY += dy * easing; } if (video.available() && (millis() > lastVideo + 10)) { lastVideo = millis(); video.read(); long startTime = millis(); elapsedTime = millis() - startTime; if (debug) { background(0); image(video, 0, 0); } else{ if (leaveTrails == false) image(testImage,0,0); if (calibrating) { image(testImage,0,0); } if (pupil != null) { ellipse(pupil.x+pupil.width/2,pupil.y+pupil.height/2,40,40); //drawX = int(map(pupil.x+pupil.width/2, 0, video.width, 0, width)); //drawY = int(map(pupil.y+pupil.height/2, 0, video.height, 0, height));*/ //drawX = adjustedPoint[0]; //drawY = adjustedPoint[1];*/ //println(adjustedPoint[0]+", "+adjustedPoint[1]); } } pupil = findPupil(); glint = null; if (pupil != null) glint = findGlint(pupil); //if the glint and the pupil are present if (glint != null && pupil != null) { // if we got both // find out where the pupil is relative to the glint int rawX = pupil.x+ pupil.width/2 - glint.x + glint.width/2; int rawY =glint.y + glint.height/2- pupil.y + pupil.height/2 ; //if the system is calibrating if (calibrator.isCurrentlyCalibrating()) { // check if we are in calibration mode Point placeToLook = calibrator.getScenePoint(); // get where they are supposed to look at //draw the dots for the user to look at for calibration purposes placeToDrawTarget = placeToLook; if (placeToDrawTarget.x > 0 &&calibrator.isCurrentlyCalibrating()) { stroke(0); fill(0); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 28, 28);// draw it so they look stroke(255); fill(255); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 22, 22);// draw it so they look stroke(40,40,255); fill(40,40,255); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 16, 16);// draw it so they look stroke(255,0,0); fill(255,0,0); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 10, 10);// draw it so they look stroke(255,255,0); fill(255,255,0); ellipse(placeToDrawTarget.x - 4, placeToDrawTarget.y - 4, 7, 7);// draw it so they look } //CHANGE THIS TO ELMO  calibrator.doCalibrationRoutine(true,rawX, rawY); println("calibrating");  } //if the system is not calibrating, else { //use the calibrator to find out where the x,y from camera corolate to on image int[] adjustedPoint = calibrator.translate(rawX, rawY); println("Adjusted x" + adjustedPoint[0] + " adjusted y" + adjustedPoint[1]);  //if the mouse is not being used, draw the crosshairs at adjusted points if(!usingMouse) { drawX = adjustedPoint[0]; drawY = adjustedPoint[1]; } }  }  }  }  public Rectangle findPupil() { // /FIND THE PUPIL ArrayList boxes = new ArrayList(); for (int row = edge; row < video.height - edge * 2; row++) { for (int col = edge; col < video.width - edge * 2; col++) { int offset = row * video.width + col; int thisPixel = video.pixels[offset]; //look for dark things if (brightness(thisPixel) < pupilThreshold) { video.pixels[offset] = 0; // be pessimistic boolean foundAHome = false; // look throught the existing for (int i = 0; i < boxes.size(); i++) { Rectangle existingBox = (Rectangle) boxes.get(i); // is this spot in an existing box Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box inflatedBox.grow(reach, reach); // widen it's reach if (inflatedBox.contains(col, row)) { existingBox.add(col, row); foundAHome = true; // no need to make a new one break; // no need to look through the rest of the boxes } } // if this does not belong to one of the existing boxes make a new one at this place if (foundAHome == false) boxes.add(new Rectangle(col, row, 0, 0)); } } }  consolidate(boxes, 0, 0);  // OF EVERYTHING YOU FIND TAKE THE ONE CLOSEST TO THE CENTER Rectangle pupil = findClosestMostBigOne(boxes, video.width / 2, video.height / 2, minPupilArea); if (debug) { // show the the edges of the search fill(0, 0, 0, 0); stroke(0, 255, 0); rect(edge, edge, video.width - 2 * edge, video.height - 2 * edge); // show all the pupil candidates stroke(0, 0, 0); for (int i = 0; i < boxes.size(); i++) { Rectangle thisBox = (Rectangle) boxes.get(i); rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height); } // show the winning pupil candidate in red if (pupil != null) { stroke(255, 0, 0); rect(pupil.x, pupil.y, pupil.width, pupil.height); } } return pupil; }  public Rectangle findGlint(Rectangle _pupil) { // ADJUST THE BOUNDS OF YOUR SEARCH TO BE AROUND AND UNER THE PUPIL int glintEdgeL = Math.max(0, _pupil.x - _pupil.width * 2); int glintEdgeR = Math.min(video.width - 1, _pupil.x + _pupil.width + _pupil.width * 2); int glintTop = _pupil.y; int glintBottom = Math.min(video.height - 1, _pupil.y + _pupil.height + _pupil.height * 2);  ArrayList glintsCandidates = new ArrayList(); for (int row = glintTop; row < glintBottom; row++) { for (int col = glintEdgeL; col < glintEdgeR; col++) { int offset = row * video.width + col; int thisPixel = video.png|none|300pxpixels[offset]; //look for bright things if (brightness(thisPixel) > glintThreshold) { if (debug) video.pixels[offset]= 0; // be pessimistic boolean foundAHome = false; // look throught the existing for (int i = 0; i < glintsCandidates.size(); i++) { Rectangle existingBox = (Rectangle) glintsCandidates.get (i); // is this spot in an existing box Rectangle inflatedBox = new Rectangle(existingBox); // copy the existing box inflatedBox.grow(reach, reach); // widen it's reach if (inflatedBox.contains(col, row)) { existingBox.add(col, row); foundAHome = true; // no need to make a new one break; // no need to look through the rest of the boxes } } // if this does not belong to one of the existing boxes make a new one at this place if (foundAHome == false) glintsCandidates.add(new Rectangle (col, row, 0, 0)); } } } // FIND THE GLINT THAT IS CLOSEST TO THE PUPIL Rectangle glint = findClosestMostBigOne(glintsCandidates,_pupil.x + _pupil.width, _pupil.y + _pupil.height / 2, 0); stroke(0, 0, 255);  if (debug) {// show all the candidate // show the edges of the search for the glint stroke(0, 75, 200); rect(glintEdgeL, glintTop, glintEdgeR - glintEdgeL, glintBottom - glintTop); for (int i = 0; i < glintsCandidates.size(); i++) { Rectangle thisBox = (Rectangle) glintsCandidates.get(i); rect(thisBox.x, thisBox.y, thisBox.width, thisBox.height); } // show the winner in red if (glint != null) { stroke(255, 0, 0); rect(glint.x, glint.y, glint.width, glint.height); } }  return glint; }  public void consolidate(ArrayList _shapes, int _consolidateReachX, int _consolidateReachY) { // check every combination of shapes for overlap // make the repeat loop backwards so you delete off the bottom of the stack for (int i = _shapes.size() - 1; i > -1; i--) { // only check the ones up Rectangle shape1 = (Rectangle) _shapes.get(i); Rectangle inflatedShape1 = new Rectangle(shape1); // copy the existing box inflatedShape1.grow(_consolidateReachX,_consolidateReachY); // widen it's reach  for (int j = i - 1; j > -1; j--) { Rectangle shape2 = (Rectangle) _shapes.get(j); if (inflatedShape1.intersects(shape2)) { shape1.add(shape2); // System.out.println("Remove" + j); _shapes.remove(j); break; } } } }  public Rectangle findClosestMostBigOne(ArrayList _allRects, int _x, int _y, int _minArea) { if (_allRects.size() == 0) return null; int winner = 0; float closest = 1000;  for (int i = 0; i < _allRects.size(); i++) { Rectangle thisRect = (Rectangle) _allRects.get(i); if (thisRect.width * thisRect.height < _minArea) continue; float thisDist = dist(_x, _y, thisRect.x + thisRect.width / 2, thisRect.y + thisRect.height / 2); if (thisDist < closest) { closest = thisDist; winner = i; } } return (Rectangle) _allRects.get(winner); } void stop() { }}