CSC400 Kinect Picture Overlay

From CSclasswiki
Jump to: navigation, search

--Thiebaut 15:37, 21 March 2012 (EDT)




Getting images

  • The image used as a background should be at least 640x480 pixels. If it is larger, reduce it to that size.
  • To convert/resize an image, use the convert utility on the Mac:
     convert fire.jpg -resize 640x480 fire.png
the command above takes fire.jpg and resizes it to be 640x480 (the size of a window of Kinect points), and stores it into a file with a new png format.



The fire.png image


The fire.png image used in the video...



The general approach

  • First an image is picked and put in the Documents folder where the kinect_xxx.out movies are located.
  • The image is resize to be 640x480 pixels (see above)
  • The program defines a PImage object called backgroundImage.
  • The setup() function opens up the image and loads its pixels in memory with
   backgroundImage.loadPixels();
  • The draw() function goes through a double for loop and gets a 5 by 5 array of pixels from the image at locations (x, y), to (x+4, y+4). These pixels are mapped to the Kinect point of coordinates (x,y).

Reference

The code

  • The highlighted parts show the code required for mapping image pixels to kinect points.
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import org.openkinect.*;
import org.openkinect.processing.*;

import processing.core.PApplet;
import processing.core.PImage;
import processing.core.PVector;
import processing.video.*;

// important!
// 1)  If you get an error about libraries, veryl likely you forgot to add -d32 to the vm arguments
//     if that is the case, click on run, run configurations, arguments, vm arguments, and enter -d32 in
//     the white box.
// 2)  If you want to increase/decrease the amount of heap space (memory space) for the application,
//     go to the same place as in 1) above and add -Xms64m -Xmx2048m after the -d32 switch.
//

public class PointCloud extends PApplet {
	
	GUIFrame guiFrame = null;
	public static int NODEPTH = 640 * 480; // # of points with depth in a frame
	String movieDirectory = System.getProperty("user.home") + "/Documents/";

	// use FakeKinect when working away from the Kinect
	// use Kinect when connected to a real Kinect
	FakeKinect kinect = new FakeKinect(this); // the kinect device
	// Kinect kinect = new Kinect(this); // the kinect device

	float Yangle = 0; // angle around y axis
	float Xangle = 0; // angle around x axis

	float lastMouseX = 0; // last position of mouse on
	float lastMouseY = 0; // screen during drag or click
	float scaleFactor = 200;

	// Size of kinect image. This is predefined by Kinect. Cannot change.
	int w = 640;
	int h = 480;
	int[] depth2 = new int[w * h];
	int[] lastFrame = new int[w * h]; // keeps track of the last frame shown
										// this way we can see what points are
										// "moving"
										// or changing fast.

	// skip factor. if 1 every point of the kinect cloud point is shown
	// if 2, every other point, etc.
	int skip = 4; // 4 by default. Good for speed

	// collection of points.
	// PointCloudCollection pointCollection = new PointCloudCollection();
	PointCloudRingBuffer pointCollection = new PointCloudRingBuffer();

	// We'll use a lookup table so that we don't have to repeat the math over
	// and over
	float[] depthLookUp = new float[2048];

	private double depthThreshold = ParameterClass.getDepthThreshold(); // controls
																		// depth
																		// threshold
																		// for
																		// displaying
	// or not displaying points on screen
	private float depthaxis = -50; // controls depth axis

	boolean autoRecording = false;
	boolean recording = false; // true if we are recording the points
	boolean playback = false; // true if we are playing the points
	boolean debug = false; // controls output of debugging info to console
	boolean displayMenu = false; // if true allows display of info on screen
	boolean recordMovie = false; // if true record a movie
	boolean turningRight = false; // turn right, camera
	boolean turningUp = false; // turn up, camera
	boolean turningLeft = false; // turn left, camera
	boolean turningDown = false; // turn down, camera

	int frameNo = 0; // counts the number of processing frames

	String keyBuffer = ""; // keyboard buffer for user command

	int currentFrameRate = 30; // rate at which frames are displayed
								// and recorded.

	// --- Threads ---
	FileReaderThread fileReaderThread = null;
	FileWriterThread fileWriterThread = null;

	// --- movie maker ---
	MovieMaker mm = null;
	String movieFileName = "";

	//--- background image for dancer ---
	PImage backgroundImage;
	
	/*
	 * debug( s ): print to console if debug flag is true
	 */
	public void debug(String s) {
		if (debug)
			System.out.println(s);
	}

	
	/*
	 ____    _____   _____   _   _   ____  
	/ ___|  | ____| |_   _| | | | | |  _ \ 
	\___ \  |  _|     | |   | | | | | |_) |
	 ___) | | |___    | |   | |_| | |  __/ 
	|____/  |_____|   |_|    \___/  |_|    
	*/
	public void setup() {
		width = 800;
		height = 600;
		size(width, height, P3D);

		// --- initialize the GUI ---
		// gui = new ControlP5GUI_Old(this);
		guiFrame = new GUIFrame(this, 300, 600);

		kinect.start();
		kinect.enableDepth(true);

		// We don't need the grayscale image in this example
		// so this makes it more efficient
		kinect.processDepthImage(false);

		// Lookup table for all possible depth values (0 - 2047)
		for (int i = 0; i < depthLookUp.length; i++) {
			depthLookUp[i] = rawDepthToMeters(i);
		}

		// set the rate to 30 frames a second
		frameRate(30);
		
		// load a background image
		backgroundImage = loadImage( movieDirectory + "fire.png" );
		//backgroundImage = loadImage( movieDirectory + "dalailama.png" );
		backgroundImage.loadPixels();
	}

	/*
		 __  __    ___    _   _   ____    _____ 
		|  \/  |  / _ \  | | | | / ___|  | ____|
		| |\/| | | | | | | | | | \___ \  |  _|  
		| |  | | | |_| | | |_| |  ___) | | |___ 
		|_|  |_|  \___/   \___/  |____/  |_____|
		
	 * When user clicks the mouse and is about to drag, use the current mouse
	 * position as the last position when we were dragging.
	 */
	public void mousePressed() {
		lastMouseX = mouseX;
		lastMouseY = mouseY;
	}

	public void mouseDragged() {
		// Rotate
		Yangle += (mouseX - lastMouseX) / 200;
		Xangle += (mouseY - lastMouseY) / 200;
		lastMouseX = mouseX;
		lastMouseY = mouseY;
	}

	/*
		 _  __          ____                             _ 
		| |/ /___ _   _|  _ \ _ __ ___  ___ ___  ___  __| |
		| ' // _ \ | | | |_) | '__/ _ \/ __/ __|/ _ \/ _` |
		| . \  __/ |_| |  __/| | |  __/\__ \__ \  __/ (_| |
		|_|\_\___|\__, |_|   |_|  \___||___/___/\___|\__,_|
		          |___/                                    
	 */
	public void keyPressed() {
		// if the key is a special key (shift, up, down, etc...)
		// do not process it
		if (key == CODED) {
			return;
		}

		// if user presses ENTER, process key buffer
		if (key == RETURN || key == ENTER) {
			processKeyBuffer();
			keyBuffer = "";
			return;
		}

		// backspace?
		else if (key == BACKSPACE || key == DELETE) {
			if (keyBuffer.length() >= 1)
				keyBuffer = keyBuffer.substring(0, keyBuffer.length() - 1);
			return;
		}

		// if there are keys already in the buffer, add the new keys in
		// without analyzing/parsing them..
		if (keyBuffer.length() > 0) {
			keyBuffer += key;
			return;
		}

		// turn right or stop right
		if (key == 'R') {
			turningRight = !turningRight;
			return;
		}

		// turn up or stop up
		if (key == 'U') {
			turningUp = !turningUp;
			return;
		}

		// turn down or stop down
		if (key == 'W') {
			turningDown = !turningDown;
			return;
		}

		// turn left or stop left
		if (key == 'L') {
			turningLeft = !turningLeft;
			return;
		}

		// if user presses +, increase scale factor by 10%
		if (key == '+') {
			scaleFactor = (float) (scaleFactor * 1.10);
		}
		// if user presses -, decrease scale factor by 10%
		else if (key == '-') {
			scaleFactor = (float) (scaleFactor * 0.90);
		}

		// menu on/off?
		else if (key == 'm') {
			//displayMenu = !displayMenu;
			if ( guiFrame.isVisible() )
				guiFrame.setVisible( false ); //hide();
			else
				guiFrame.setVisible( true ); // show();
		}
		// change depth threshold for showing points
		else if (key == 'd') {
			println( "Deprecated command.  Use GUI Menu!" );
			//depthThreshold = depthThreshold * 0.995;
			//ParameterClass.setDepthThreshold(depthThreshold);
		} else if (key == 'D') {
			println( "Deprecated command.  Use GUI Menu!" );
			//depthThreshold = depthThreshold * 1.005;
			//ParameterClass.setDepthThreshold(depthThreshold);
		}
		// change depthaxis
		else if (key == 'i')
			println( "Deprecated command.  Use GUI Menu!" );
			//depthaxis = (float) (depthaxis * 0.95);
		else if (key == 'o')
			println( "Deprecated command.  Use GUI Menu!" );
			//depthaxis = (float) (depthaxis * 1.05);

		// auto-recording?
		else if (key == 'a' || key == 'A') {
			autoRecording = !autoRecording;
		}

		// stop playback
		else if (key == 't') {
			playback = false;
		}

		// change frameRate
		else if (key == 'f') {
			currentFrameRate -= 1;
			frameRate(currentFrameRate);
		} else if (key == 'F') {
			currentFrameRate += 1;
			frameRate(currentFrameRate);
		}

		// change the skip factor
		else if (key == 'k')
			skip = Math.max(1, skip - 1);
		else if (key == 'K')
			skip = (skip + 1);

		// otherwise the user is typing a command
		else {
			keyBuffer += key;
		}
	}

	public void updateDepthAxis( int value ) {
		depthaxis = value;
		ParameterClass.setDepthAxis( depthaxis );
	}

	public void updateDepthThreshold( int value ) {
		depthThreshold = value;
		ParameterClass.setDepthThreshold(depthThreshold);
	}

	public void updateAutoRecord( boolean x ) {
		autoRecording = x;
		ParameterClass.setAutoRecording( x );
	}
	/*
	 * processKeyBuffer(): called when user presses the ENTER key after a series
	 * of keystrokes. Processes contents of keyBuffer and attempts to decode the
	 * command entered by the user. Current commands include: p nnn where nnn is
	 * an integer. playback a recorded movie with index nnn
	 */
	private void processKeyBuffer() {
		if (keyBuffer.length() == 0)
			return;

		String[] tokens = keyBuffer.split(" ");
		int index;

		if (tokens.length == 0) {
			println("Invalid command... skipping.");
			keyBuffer = "";
			return;
		}

		// Playback command?
		if (tokens[0].equals("p") || tokens[0].equals("pc")
				|| tokens[0].equals("cp")) {
			if (tokens.length < 2) {
				System.out
						.println("*** Error: you must supply a file index to play back ***");
				keyBuffer = "";
				return;
			}
			try {
				index = Integer.parseInt(tokens[1], 10);
			} catch (NumberFormatException e) {
				System.out
						.println("*** ProcessKeyBuffer Error*** Invalid command");
				keyBuffer = "";
				return;
			}
			println("===> Executing command: playback movie #" + index);
			while (fileReaderThread != null && fileReaderThread.isAlive()) {
				fileReaderThread.finish();
				debug("waiting for fileReaderThread to finish...");
				try {
					Thread.sleep(100); // wait 1/2 second for thread to
										// terminate
				} catch (InterruptedException e) {
				}
			}
			// --- clear buffer ---
			pointCollection.resetBuffer();
			fileReaderThread = new FileReaderThread(this, pointCollection,
					index, pointCollection.CHUNKSIZE);
			fileReaderThread.setContinuous(false); // run once only
			if (tokens[0].equals("pc") || tokens[0].equals("cp"))
				fileReaderThread.setContinuous(true); // run continuously
			fileReaderThread.start();
			debug("fileReaderThread started!");
			playback = true;
			keyBuffer = "";
			return;
		}

		// record a movie? if command is "M ssss", then ssss is filename for
		// storing the movie in. If command is "M" then stop recording movie
		if (tokens[0].equals("M")) {
			// filename after "M"?
			if (tokens.length > 1) {
				movieFileName = movieDirectory + tokens[1] + ".mov";
				println("recording movie " + movieFileName);
				mm = new MovieMaker(this, width, height, movieFileName, 30,
						MovieMaker.ANIMATION, MovieMaker.MEDIUM /* BEST */);
				recordMovie = true;
				displayMenu = false;
			} else {
				if (mm != null) {
					recordMovie = false;
					mm.finish();
					// give program about a second to close movie
					delay(1000);
					mm = null;
					println("stopped recording movie");
				}
			}
		}

		// Recording command?
		if (tokens[0].equals("r")) {

			// stop playback
			playback = false;

			// empty frame buffer for recording
			pointCollection.resetBuffer(); // position at beginning of buffer

			// did the user type a number after the 'r'?
			if (tokens.length > 1)
				index = Integer.parseInt(tokens[1], 10);
			else {
				index = ParameterClass.getFileIndex();
				ParameterClass.incrementFileIndex();
			}
			println("===> Executing command: record to movie #" + index);
			pointCollection.setFileIndex(index);

			// --- delete output file if it exists ---
			pointCollection.deleteFile();

			// --- create a new thread and start it ---
			while (fileWriterThread != null && fileWriterThread.isAlive()) {
				fileWriterThread.finish();
				try {
					// wait 0.1 sec and test again
					Thread.sleep(100);
				} catch (InterruptedException e) {
				}
			}
			fileWriterThread = new FileWriterThread(pointCollection);
			fileWriterThread.start();
			recording = true;
			keyBuffer = "";
			return;
		}

		// stop recording?
		if (tokens[0].equals("s")) {
			// --- stop thread and wait till it's done ---
			while (fileWriterThread != null && fileWriterThread.isAlive()) {
				fileWriterThread.finish();
				try {
					// wait 0.1 sec and test again
					Thread.sleep(100);
				} catch (InterruptedException e) {
				}
			}
			fileWriterThread = null;
			recording = false;

			keyBuffer = "";
			return;
		}

		// clear keyBuffer...
		keyBuffer = "";
	}

	/*
	 */
	public void displayMenu() {
		if (!displayMenu)
			return;
		int y = 15;
		textMode(SCREEN);
		text("Frame #" + frameNo, 10, y);
		y += 15;
		text("-------------------------------", 10, y);
		y += 15;
		text("Kinect FR: " + (int) kinect.getDepthFPS(), 10, y);
		y += 15;
		text("Processing FR: " + (int) frameRate, 10, y);
		y += 15;
		text("Buffer: " + keyBuffer, 10, y);
		y += 15;
		text("-------------------------------", 10, y);
		y += 15;
		text("Commands:", 10, y);
		y += 15;
		text(" +/-   zoom in/out", 10, y);
		y += 15;
		text(" R     go right or stop right", 10, y);
		y += 15;
		text(" L     go left or stop left", 10, y);
		y += 15;
		text(" W     go down or stop down", 10, y);
		y += 15;
		text(" U     go up or stop up", 10, y);
		y += 15;
		text(" d/D   change depth", 10, y);
		y += 15;
		text(" i/o   change depth axis", 10, y);
		y += 15;
		text(" a     start/stop auto recording mode", 10, y);
		y += 15;
		text(" t     stop playback", 10, y);
		y += 15;
		text(" k/K   decrease/increase the skip factor", 10, y);
		y += 15;
		text(" r     record to next available file", 10, y);
		y += 15;
		text(" r n   record to file number n", 10, y);
		y += 15;
		text(" s     stop recording", 10, y);
		y += 15;
		text(" p n   play file number n", 10, y);
		y += 15;
		text(" pc nn play continuously file number nn", 10, y);
		y += 15;
		text(" f/F   decreases/increases frame rate", 10, y);
		y += 15;
		text(" m     turn menu on/off", 10, y);
		y += 15;
		text(" M ssss start recording movie to file ssss", 10, y);
		y += 15;
		text(" M     stop recording movie", 10, y);
		y += 15;
	}

	// Turn Playback ON/OFF
	public void setPlayback(boolean b) {
		playback = b;
	}


	/*
			 ____    ____       _     __        __
			|  _ \  |  _ \     / \    \ \      / /
			| | | | | |_) |   / _ \    \ \ /\ / / 
			| |_| | |  _ <   / ___ \    \ V  V /  
			|____/  |_| \_\ /_/   \_\    \_/\_/   
	 */
	public void draw() {

		if (turningRight == true)
			Yangle -= 0.003;

		if (turningLeft == true)
			Yangle += 0.003;

		if (turningUp == true)
			Xangle -= 0.003;

		if (turningDown == true)
			Xangle += 0.003;

		background(0);
		fill(255);
		//displayMenu();
		
		//--- set quantities in SharedData static class for other applet GUI to pick up ---
		guiFrame.setFrameCounter( frameNo );
		guiFrame.setProcessingFrameCounter((int) frameRate );
		guiFrame.setKinectFrames( (int) kinect.getDepthFPS() );

		//--- save the matrix before we sart doing 3D suff ---
		pushMatrix();
		
		// blinking red dot if recording
		if (recording && (frameNo / 15) % 2 == 0) {
			fill(255, 0, 0); // red
			ellipse(width - 30, 30, 20, 20); // top right corner
		}
		if (autoRecording) {
			fill(255, 255, 0);
			ellipse(width - 30, 60, 20, 20);
		}

		// Get the raw depth as array of integers
		int[] depth = kinect.getRawDepth();

		// are we recording?
		if (recording == true) {

			// is there still room in the buffer?
			if (!pointCollection.isFull()) {
				pointCollection.addNewFrame(depth);
			}
		}

		// playback or real kinect?
		// if playback, take the last frame from ring buffer and put it into
		// depth array
		if (playback == true) {
			if (pointCollection.currentFrameIsFirstFrame())
				frameNo = 0;
			pointCollection.getNewFrameInto(depth2);
			for (int i = 0; i < NODEPTH; i++) {
				if (depth2[i] < depth[i]) {
					depth[i] = depth2[i];
				}
			}
		}

		// We're just going to calculate and draw every 4th pixel (equivalent of
		// 160x120)

		// Translate and rotate
		translate(width / 2, height / 4, depthaxis);
		rotateY(Yangle);
		rotateX(Xangle);

		for (int x = 0; x < w; x += skip) {
			for (int y = 0; y < h; y += skip) {
				int offset = x + y * w;

				// Convert kinect data to world xyz coordinate
				if (depth[offset] == 0)
					continue;

				int rawDepth = depth[offset];
				/*
				 * if (rawDepth == 0) { continue; }
				 */
				// block out the background
				if (rawDepth > depthThreshold)
					continue;

				PVector v = depthToWorld(x, y, rawDepth);

				pushMatrix();
				translate(v.x * scaleFactor, v.y * scaleFactor, scaleFactor
						- v.z * scaleFactor);

				// DFT Note: in the code below, change the if ( true ) test to
				// if (false )
				// to make the color proportional to the amount of movements the
				// points experience. If a point moves fast it becomes a bright
				// color, otherwise it gets close to dark
				//
				// Draw a point. Apply color to point
				int col;
				if (true) {
					// the old way
					col = color(255 - rawDepth / 8, rawDepth / 8, rawDepth / 8);
				} else {
					// compute color based on movement
					// compute by how much the point has moved since the last
					// frame
					int diff = depth[offset] - lastFrame[offset];
					lastFrame[offset] = depth[offset];

					/*
					 * if ( diff < 0 ) { // the point is going toward the viewer
					 * // enhance the speed of the point and make it a color col
					 * = -diff * 20; // compute a color that is mostly black if
					 * no movement col = color( col + 10, col/2+20, col/4 + 50
					 * ); } else { // the point is going away from the viewer //
					 * enhance the speed of the point and make it a color col =
					 * diff * 20; col = color( col/2+20, col + 10, col/4 + 50 );
					 * }
					 */

					if (abs(diff) < 10) {
						col = color(0, 0, 0);
					} else {
						col = color(255, 0, 0);
					}
				}

				
				// set color of the point (or rectangle, or ellipse)
				//stroke(col);
				//fill(col);
				// draw a single point
				//point(0, 0);

				// take a 5x5 square from the background image and map it to the Kinect pixel
				int imageOffset = x + y * backgroundImage.width; // backgroundImage.width;
				if (x < backgroundImage.width - 5
						&& y < backgroundImage.height - 5) {
					for (int xx = 0; xx < 5; xx += 1)
						for (int yy = 0; yy < 5; yy += 1) {
							float r = red(backgroundImage.pixels[imageOffset
									+ xx + yy * backgroundImage.width]);
							float g = green(backgroundImage.pixels[imageOffset
									+ xx + yy * backgroundImage.width]);
							float b = blue(backgroundImage.pixels[imageOffset
									+ xx + yy * backgroundImage.width]);
							stroke(color(r, g, b));
							point(xx, yy);
						}
				}
				
				// draw a circle
				// ellipse( 0, 0, 10, 10 );

				popMatrix();
			}
		}

		// update frame counter
		frameNo += 1;

		// record this frame if playing movie...
		if (recordMovie) {
			mm.addFrame();
		}
		
		//--- force a call to gui.draw() ---
		/* INFO: You are using renderer processing.core.PGraphics3D.
		In order to render controlP5 elements you need to call the ControlP5's draw() manually.
		Suggestion is to put controlP5.draw(); at the bottom of the draw function of your sketch.
		*/
		popMatrix();

		//--- redraw second window ---
		SharedData.kinectFrames 			=  (int) kinect.getDepthFPS();
		SharedData.frameCounter 			= frameNo;
		SharedData.ProcessingFrameCounter 	= frameRate;

		guiFrame.redraw();
	}

	// These functions come from:
	// http://graphics.stanford.edu/~mdfisher/Kinect.html
	float rawDepthToMeters(int depthValue) {
		if (depthValue < 2047) {
			return (float) (1.0 / ((double) (depthValue) * -0.0030711016 + 3.3309495161));
		}
		return 0.0f;
	}

	PVector depthToWorld(int x, int y, int depthValue) {

		final double fx_d = 1.0 / 5.9421434211923247e+02;
		final double fy_d = 1.0 / 5.9104053696870778e+02;
		final double cx_d = 3.3930780975300314e+02;
		final double cy_d = 2.4273913761751615e+02;

		PVector result = new PVector();
		double depth = depthLookUp[depthValue];// rawDepthToMeters(depthValue);
		result.x = (float) ((x - cx_d) * depth * fx_d);
		result.y = (float) ((y - cy_d) * depth * fy_d);
		result.z = (float) (depth);
		return result;
	}

	public void stop() {
		kinect.quit();
		super.stop();
	}

}


Blur and Dilate Test


import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;

import org.openkinect.*;
import org.openkinect.processing.*;

import processing.core.PApplet;
import processing.core.PImage;
import processing.core.PVector;
import processing.video.*;

// important!
// 1)  If you get an error about libraries, veryl likely you forgot to add -d32 to the vm arguments
//     if that is the case, click on run, run configurations, arguments, vm arguments, and enter -d32 in
//     the white box.
// 2)  If you want to increase/decrease the amount of heap space (memory space) for the application,
//     go to the same place as in 1) above and add -Xms64m -Xmx2048m after the -d32 switch.
//

public class PointCloud extends PApplet {
	
	GUIFrame guiFrame = null;
	public static int NODEPTH = 640 * 480; // # of points with depth in a frame
	String movieDirectory = System.getProperty("user.home") + "/Documents/";

	// use FakeKinect when working away from the Kinect
	// use Kinect when connected to a real Kinect
	FakeKinect kinect = new FakeKinect(this); // the kinect device
	// Kinect kinect = new Kinect(this); // the kinect device

	float Yangle = 0; // angle around y axis
	float Xangle = 0; // angle around x axis

	float lastMouseX = 0; // last position of mouse on
	float lastMouseY = 0; // screen during drag or click
	float scaleFactor = 200;

	// Size of kinect image. This is predefined by Kinect. Cannot change.
	int w = 640;
	int h = 480;
	int[] depth2 = new int[w * h];
	int[] lastFrame = new int[w * h]; // keeps track of the last frame shown
										// this way we can see what points are
										// "moving"
										// or changing fast.

	// skip factor. if 1 every point of the kinect cloud point is shown
	// if 2, every other point, etc.
	int skip = 4; // 4 by default. Good for speed

	// collection of points.
	// PointCloudCollection pointCollection = new PointCloudCollection();
	PointCloudRingBuffer pointCollection = new PointCloudRingBuffer();

	// We'll use a lookup table so that we don't have to repeat the math over
	// and over
	float[] depthLookUp = new float[2048];

	private double depthThreshold = ParameterClass.getDepthThreshold(); // controls
																		// depth
																		// threshold
																		// for
																		// displaying
	// or not displaying points on screen
	private float depthaxis = -50; // controls depth axis

	boolean autoRecording = false;
	boolean recording = false; // true if we are recording the points
	boolean playback = false; // true if we are playing the points
	boolean debug = false; // controls output of debugging info to console
	boolean displayMenu = false; // if true allows display of info on screen
	boolean recordMovie = false; // if true record a movie
	boolean turningRight = false; // turn right, camera
	boolean turningUp = false; // turn up, camera
	boolean turningLeft = false; // turn left, camera
	boolean turningDown = false; // turn down, camera

	int frameNo = 0; // counts the number of processing frames

	String keyBuffer = ""; // keyboard buffer for user command

	int currentFrameRate = 30; // rate at which frames are displayed
								// and recorded.

	// --- Threads ---
	FileReaderThread fileReaderThread = null;
	FileWriterThread fileWriterThread = null;

	// --- movie maker ---
	MovieMaker mm = null;
	String movieFileName = "";

	//--- background image for dancer ---
	PImage backgroundImage;
	int backgroundImageIndex = 0;
	
	/*
	 * debug( s ): print to console if debug flag is true
	 */
	public void debug(String s) {
		if (debug)
			System.out.println(s);
	}

	
	/*
	 ____    _____   _____   _   _   ____  
	/ ___|  | ____| |_   _| | | | | |  _ \ 
	\___ \  |  _|     | |   | | | | | |_) |
	 ___) | | |___    | |   | |_| | |  __/ 
	|____/  |_____|   |_|    \___/  |_|    
	*/
	public void setup() {
		width = 800;
		height = 600;
		size(width, height, P3D);

		// --- initialize the GUI ---
		// gui = new ControlP5GUI_Old(this);
		guiFrame = new GUIFrame(this, 300, 600);

		kinect.start();
		kinect.enableDepth(true);

		// We don't need the grayscale image in this example
		// so this makes it more efficient
		kinect.processDepthImage(false);

		// Lookup table for all possible depth values (0 - 2047)
		for (int i = 0; i < depthLookUp.length; i++) {
			depthLookUp[i] = rawDepthToMeters(i);
		}

		// set the rate to 30 frames a second
		frameRate(30);
		
		// load a background image
		backgroundImage = loadImage( movieDirectory + "fire.png" );
		//backgroundImage = loadImage( movieDirectory + "dalailama.png" );
		
		backgroundImage.loadPixels();
		
		//--- black background... ---
		background(0);
	}

	/*
		 __  __    ___    _   _   ____    _____ 
		|  \/  |  / _ \  | | | | / ___|  | ____|
		| |\/| | | | | | | | | | \___ \  |  _|  
		| |  | | | |_| | | |_| |  ___) | | |___ 
		|_|  |_|  \___/   \___/  |____/  |_____|
		
	 * When user clicks the mouse and is about to drag, use the current mouse
	 * position as the last position when we were dragging.
	 */
	public void mousePressed() {
		lastMouseX = mouseX;
		lastMouseY = mouseY;
	}

	public void mouseDragged() {
		// Rotate
		Yangle += (mouseX - lastMouseX) / 200;
		Xangle += (mouseY - lastMouseY) / 200;
		lastMouseX = mouseX;
		lastMouseY = mouseY;
	}

	/*
		 _  __          ____                             _ 
		| |/ /___ _   _|  _ \ _ __ ___  ___ ___  ___  __| |
		| ' // _ \ | | | |_) | '__/ _ \/ __/ __|/ _ \/ _` |
		| . \  __/ |_| |  __/| | |  __/\__ \__ \  __/ (_| |
		|_|\_\___|\__, |_|   |_|  \___||___/___/\___|\__,_|
		          |___/                                    
	 */
	public void keyPressed() {
		// if the key is a special key (shift, up, down, etc...)
		// do not process it
		if (key == CODED) {
			return;
		}

		// if user presses ENTER, process key buffer
		if (key == RETURN || key == ENTER) {
			processKeyBuffer();
			keyBuffer = "";
			return;
		}

		// backspace?
		else if (key == BACKSPACE || key == DELETE) {
			if (keyBuffer.length() >= 1)
				keyBuffer = keyBuffer.substring(0, keyBuffer.length() - 1);
			return;
		}

		// if there are keys already in the buffer, add the new keys in
		// without analyzing/parsing them..
		if (keyBuffer.length() > 0) {
			keyBuffer += key;
			return;
		}

		// turn right or stop right
		if (key == 'R') {
			turningRight = !turningRight;
			return;
		}

		// turn up or stop up
		if (key == 'U') {
			turningUp = !turningUp;
			return;
		}

		// turn down or stop down
		if (key == 'W') {
			turningDown = !turningDown;
			return;
		}

		// turn left or stop left
		if (key == 'L') {
			turningLeft = !turningLeft;
			return;
		}

		// if user presses +, increase scale factor by 10%
		if (key == '+') {
			scaleFactor = (float) (scaleFactor * 1.10);
		}
		// if user presses -, decrease scale factor by 10%
		else if (key == '-') {
			scaleFactor = (float) (scaleFactor * 0.90);
		}

		// menu on/off?
		else if (key == 'm') {
			//displayMenu = !displayMenu;
			if ( guiFrame.isVisible() )
				guiFrame.setVisible( false ); //hide();
			else
				guiFrame.setVisible( true ); // show();
		}
		// change depth threshold for showing points
		else if (key == 'd') {
			println( "Deprecated command.  Use GUI Menu!" );
			//depthThreshold = depthThreshold * 0.995;
			//ParameterClass.setDepthThreshold(depthThreshold);
		} else if (key == 'D') {
			println( "Deprecated command.  Use GUI Menu!" );
			//depthThreshold = depthThreshold * 1.005;
			//ParameterClass.setDepthThreshold(depthThreshold);
		}
		// change depthaxis
		else if (key == 'i')
			println( "Deprecated command.  Use GUI Menu!" );
			//depthaxis = (float) (depthaxis * 0.95);
		else if (key == 'o')
			println( "Deprecated command.  Use GUI Menu!" );
			//depthaxis = (float) (depthaxis * 1.05);

		// auto-recording?
		else if (key == 'a' || key == 'A') {
			autoRecording = !autoRecording;
		}

		// stop playback
		else if (key == 't') {
			playback = false;
		}

		// change frameRate
		else if (key == 'f') {
			currentFrameRate -= 1;
			frameRate(currentFrameRate);
		} else if (key == 'F') {
			currentFrameRate += 1;
			frameRate(currentFrameRate);
		}

		// change the skip factor
		else if (key == 'k')
			skip = Math.max(1, skip - 1);
		else if (key == 'K')
			skip = (skip + 1);

		// otherwise the user is typing a command
		else {
			keyBuffer += key;
		}
	}

	public void updateDepthAxis( int value ) {
		depthaxis = value;
		ParameterClass.setDepthAxis( depthaxis );
	}

	public void updateDepthThreshold( int value ) {
		depthThreshold = value;
		ParameterClass.setDepthThreshold(depthThreshold);
	}

	public void updateAutoRecord( boolean x ) {
		autoRecording = x;
		ParameterClass.setAutoRecording( x );
	}
	/*
	 * processKeyBuffer(): called when user presses the ENTER key after a series
	 * of keystrokes. Processes contents of keyBuffer and attempts to decode the
	 * command entered by the user. Current commands include: p nnn where nnn is
	 * an integer. playback a recorded movie with index nnn
	 */
	private void processKeyBuffer() {
		if (keyBuffer.length() == 0)
			return;

		String[] tokens = keyBuffer.split(" ");
		int index;

		if (tokens.length == 0) {
			println("Invalid command... skipping.");
			keyBuffer = "";
			return;
		}

		// Playback command?
		if (tokens[0].equals("p") || tokens[0].equals("pc")
				|| tokens[0].equals("cp")) {
			if (tokens.length < 2) {
				System.out
						.println("*** Error: you must supply a file index to play back ***");
				keyBuffer = "";
				return;
			}
			try {
				index = Integer.parseInt(tokens[1], 10);
			} catch (NumberFormatException e) {
				System.out
						.println("*** ProcessKeyBuffer Error*** Invalid command");
				keyBuffer = "";
				return;
			}
			println("===> Executing command: playback movie #" + index);
			while (fileReaderThread != null && fileReaderThread.isAlive()) {
				fileReaderThread.finish();
				debug("waiting for fileReaderThread to finish...");
				try {
					Thread.sleep(100); // wait 1/2 second for thread to
										// terminate
				} catch (InterruptedException e) {
				}
			}
			// --- clear buffer ---
			pointCollection.resetBuffer();
			fileReaderThread = new FileReaderThread(this, pointCollection,
					index, pointCollection.CHUNKSIZE);
			fileReaderThread.setContinuous(false); // run once only
			if (tokens[0].equals("pc") || tokens[0].equals("cp"))
				fileReaderThread.setContinuous(true); // run continuously
			fileReaderThread.start();
			debug("fileReaderThread started!");
			playback = true;
			keyBuffer = "";
			return;
		}

		// record a movie? if command is "M ssss", then ssss is filename for
		// storing the movie in. If command is "M" then stop recording movie
		if (tokens[0].equals("M")) {
			// filename after "M"?
			if (tokens.length > 1) {
				movieFileName = movieDirectory + tokens[1] + ".mov";
				println("recording movie " + movieFileName);
				mm = new MovieMaker(this, width, height, movieFileName, 30,
						MovieMaker.ANIMATION, MovieMaker.MEDIUM /* BEST */);
				recordMovie = true;
				displayMenu = false;
			} else {
				if (mm != null) {
					recordMovie = false;
					mm.finish();
					// give program about a second to close movie
					delay(1000);
					mm = null;
					println("stopped recording movie");
				}
			}
		}

		// Recording command?
		if (tokens[0].equals("r")) {

			// stop playback
			playback = false;

			// empty frame buffer for recording
			pointCollection.resetBuffer(); // position at beginning of buffer

			// did the user type a number after the 'r'?
			if (tokens.length > 1)
				index = Integer.parseInt(tokens[1], 10);
			else {
				index = ParameterClass.getFileIndex();
				ParameterClass.incrementFileIndex();
			}
			println("===> Executing command: record to movie #" + index);
			pointCollection.setFileIndex(index);

			// --- delete output file if it exists ---
			pointCollection.deleteFile();

			// --- create a new thread and start it ---
			while (fileWriterThread != null && fileWriterThread.isAlive()) {
				fileWriterThread.finish();
				try {
					// wait 0.1 sec and test again
					Thread.sleep(100);
				} catch (InterruptedException e) {
				}
			}
			fileWriterThread = new FileWriterThread(pointCollection);
			fileWriterThread.start();
			recording = true;
			keyBuffer = "";
			return;
		}

		// stop recording?
		if (tokens[0].equals("s")) {
			// --- stop thread and wait till it's done ---
			while (fileWriterThread != null && fileWriterThread.isAlive()) {
				fileWriterThread.finish();
				try {
					// wait 0.1 sec and test again
					Thread.sleep(100);
				} catch (InterruptedException e) {
				}
			}
			fileWriterThread = null;
			recording = false;

			keyBuffer = "";
			return;
		}

		// clear keyBuffer...
		keyBuffer = "";
	}

	/*
	 */
	public void displayMenu() {
		if (!displayMenu)
			return;
		int y = 15;
		textMode(SCREEN);
		text("Frame #" + frameNo, 10, y);
		y += 15;
		text("-------------------------------", 10, y);
		y += 15;
		text("Kinect FR: " + (int) kinect.getDepthFPS(), 10, y);
		y += 15;
		text("Processing FR: " + (int) frameRate, 10, y);
		y += 15;
		text("Buffer: " + keyBuffer, 10, y);
		y += 15;
		text("-------------------------------", 10, y);
		y += 15;
		text("Commands:", 10, y);
		y += 15;
		text(" +/-   zoom in/out", 10, y);
		y += 15;
		text(" R     go right or stop right", 10, y);
		y += 15;
		text(" L     go left or stop left", 10, y);
		y += 15;
		text(" W     go down or stop down", 10, y);
		y += 15;
		text(" U     go up or stop up", 10, y);
		y += 15;
		text(" d/D   change depth", 10, y);
		y += 15;
		text(" i/o   change depth axis", 10, y);
		y += 15;
		text(" a     start/stop auto recording mode", 10, y);
		y += 15;
		text(" t     stop playback", 10, y);
		y += 15;
		text(" k/K   decrease/increase the skip factor", 10, y);
		y += 15;
		text(" r     record to next available file", 10, y);
		y += 15;
		text(" r n   record to file number n", 10, y);
		y += 15;
		text(" s     stop recording", 10, y);
		y += 15;
		text(" p n   play file number n", 10, y);
		y += 15;
		text(" pc nn play continuously file number nn", 10, y);
		y += 15;
		text(" f/F   decreases/increases frame rate", 10, y);
		y += 15;
		text(" m     turn menu on/off", 10, y);
		y += 15;
		text(" M ssss start recording movie to file ssss", 10, y);
		y += 15;
		text(" M     stop recording movie", 10, y);
		y += 15;
	}

	// Turn Playback ON/OFF
	public void setPlayback(boolean b) {
		playback = b;
	}


	/*
			 ____    ____       _     __        __
			|  _ \  |  _ \     / \    \ \      / /
			| | | | | |_) |   / _ \    \ \ /\ / / 
			| |_| | |  _ <   / ___ \    \ V  V /  
			|____/  |_| \_\ /_/   \_\    \_/\_/   
	 */
	public void draw() {
		//--- make the background image rotate left to right ---
		backgroundImageIndex = ( backgroundImageIndex + 5 ) % backgroundImage.width;
		
		if (turningRight == true)
			Yangle -= 0.003;

		if (turningLeft == true)
			Yangle += 0.003;

		if (turningUp == true)
			Xangle -= 0.003;

		if (turningDown == true)
			Xangle += 0.003;

		//background(0);
		filter(DILATE);
		filter(BLUR);
		fill(255);
		//displayMenu();
		
		//--- set quantities in SharedData static class for other applet GUI to pick up ---
		guiFrame.setFrameCounter( frameNo );
		guiFrame.setProcessingFrameCounter((int) frameRate );
		guiFrame.setKinectFrames( (int) kinect.getDepthFPS() );

		//--- save the matrix before we sart doing 3D suff ---
		pushMatrix();
		
		// blinking red dot if recording
		if (recording && (frameNo / 15) % 2 == 0) {
			fill(255, 0, 0); // red
			ellipse(width - 30, 30, 20, 20); // top right corner
		}
		if (autoRecording) {
			fill(255, 255, 0);
			ellipse(width - 30, 60, 20, 20);
		}

		// Get the raw depth as array of integers
		int[] depth = kinect.getRawDepth();

		// are we recording?
		if (recording == true) {

			// is there still room in the buffer?
			if (!pointCollection.isFull()) {
				pointCollection.addNewFrame(depth);
			}
		}

		// playback or real kinect?
		// if playback, take the last frame from ring buffer and put it into
		// depth array
		if (playback == true) {
			if (pointCollection.currentFrameIsFirstFrame())
				frameNo = 0;
			pointCollection.getNewFrameInto(depth2);
			for (int i = 0; i < NODEPTH; i++) {
				if (depth2[i] < depth[i]) {
					depth[i] = depth2[i];
				}
			}
		}

		// We're just going to calculate and draw every 4th pixel (equivalent of
		// 160x120)

		// Translate and rotate
		translate(width / 2, height / 4, depthaxis);
		rotateY(Yangle);
		rotateX(Xangle);
		for (int x = 0; x < w; x += skip) {
			for (int y = 0; y < h; y += skip) {
				int offset = x + y * w;

				// Convert kinect data to world xyz coordinate
				if (depth[offset] == 0)
					continue;

				int rawDepth = depth[offset];
				/*
				 * if (rawDepth == 0) { continue; }
				 */
				// block out the background
				if (rawDepth > depthThreshold)
					continue;

				PVector v = depthToWorld(x, y, rawDepth);

				pushMatrix();
				translate(v.x * scaleFactor, v.y * scaleFactor, scaleFactor
						- v.z * scaleFactor);

				// DFT Note: in the code below, change the if ( true ) test to
				// if (false )
				// to make the color proportional to the amount of movements the
				// points experience. If a point moves fast it becomes a bright
				// color, otherwise it gets close to dark
				//
				// Draw a point. Apply color to point
				int col;
				if (true) {
					// the old way
					col = color(255 - rawDepth / 8, rawDepth / 8, rawDepth / 8);
				} else {
					// compute color based on movement
					// compute by how much the point has moved since the last
					// frame
					int diff = depth[offset] - lastFrame[offset];
					lastFrame[offset] = depth[offset];

					/*
					 * if ( diff < 0 ) { // the point is going toward the viewer
					 * // enhance the speed of the point and make it a color col
					 * = -diff * 20; // compute a color that is mostly black if
					 * no movement col = color( col + 10, col/2+20, col/4 + 50
					 * ); } else { // the point is going away from the viewer //
					 * enhance the speed of the point and make it a color col =
					 * diff * 20; col = color( col/2+20, col + 10, col/4 + 50 );
					 * }
					 */

					if (abs(diff) < 10) {
						col = color(0, 0, 0);
					} else {
						col = color(255, 0, 0);
					}
				}

				
				// set color of the point (or rectangle, or ellipse)
				//stroke(col);
				//fill(col);
				// draw a single point
				//point(0, 0);

				// take a 5x5 square from the background image and map it to the Kinect pixel
				int imageOffset = ( backgroundImageIndex + x ) % backgroundImage.width + y * backgroundImage.width;
				if (x < backgroundImage.width - 5
						&& y < backgroundImage.height - 5) {
					for (int xx = 0; xx < 3; xx += 1)
						for (int yy = 0; yy < 3; yy += 1) {
							float r = red(backgroundImage.pixels[imageOffset
									+ xx + yy * backgroundImage.width]);
							float g = green(backgroundImage.pixels[imageOffset
									+ xx + yy * backgroundImage.width]);
							float b = blue(backgroundImage.pixels[imageOffset
									+ xx + yy * backgroundImage.width]);
							stroke(color(r, g, b));
							point(xx, yy);
						}
				}
				
				// draw a circle
				// ellipse( 0, 0, 10, 10 );

				popMatrix();
			}
		}

		// update frame counter
		frameNo += 1;

		// record this frame if playing movie...
		if (recordMovie) {
			mm.addFrame();
		}
		
		//--- force a call to gui.draw() ---
		/* INFO: You are using renderer processing.core.PGraphics3D.
		In order to render controlP5 elements you need to call the ControlP5's draw() manually.
		Suggestion is to put controlP5.draw(); at the bottom of the draw function of your sketch.
		*/
		popMatrix();

		//--- redraw second window ---
		SharedData.kinectFrames 			=  (int) kinect.getDepthFPS();
		SharedData.frameCounter 			= frameNo;
		SharedData.ProcessingFrameCounter 	= frameRate;

		guiFrame.redraw();
	}

	// These functions come from:
	// http://graphics.stanford.edu/~mdfisher/Kinect.html
	float rawDepthToMeters(int depthValue) {
		if (depthValue < 2047) {
			return (float) (1.0 / ((double) (depthValue) * -0.0030711016 + 3.3309495161));
		}
		return 0.0f;
	}

	PVector depthToWorld(int x, int y, int depthValue) {

		final double fx_d = 1.0 / 5.9421434211923247e+02;
		final double fy_d = 1.0 / 5.9104053696870778e+02;
		final double cx_d = 3.3930780975300314e+02;
		final double cy_d = 2.4273913761751615e+02;

		PVector result = new PVector();
		double depth = depthLookUp[depthValue];// rawDepthToMeters(depthValue);
		result.x = (float) ((x - cx_d) * depth * fx_d);
		result.y = (float) ((y - cy_d) * depth * fy_d);
		result.z = (float) (depth);
		return result;
	}

	public void stop() {
		kinect.quit();
		super.stop();
	}

}


Fast Blur Function


                //--- apply filtering only every other call to draw() ---
		if ( pingPong ) {
			//background(0);
			//filter( DILATE );
			//filter( POSTERIZE, 4 );
			//filter( BLUR, 1 );
			fastBlur( 5 );
			filter( ERODE );
		}
		pingPong = !pingPong;


    • fastBlur( int radius ):


	void fastBlur( int radius ){
		
		  if (radius<1){
		    return;
		  }
		  loadPixels();
		  int w=width;
		  int h=height;
		  int wm=w-1;
		  int hm=h-1;
		  int wh=w*h;
		  int div=radius+radius+1;
		  int a[]=new int[wh]; // i've added this
		  int r[]=new int[wh];
		  int g[]=new int[wh];
		  int b[]=new int[wh];
		  int asum,rsum,gsum,bsum,x,y,i,p,p1,p2,yp,yi,yw; // and the asum here
		  int vmin[] = new int[max(w,h)];
		  int vmax[] = new int[max(w,h)];
		  int[] pix=pixels;
		  int dv[]=new int[256*div];
		  for (i=0;i<256*div;i++){
		     dv[i]=(i/div);
		  }

		  yw=yi=0;

		  for (y=0;y<h;y++){
		    asum=rsum=gsum=bsum=0; // asum
		    for(i=-radius;i<=radius;i++){
			p=pix[yi+min(wm,max(i,0))];
			asum+=(p>>24) & 0xff;
			rsum+=(p & 0xff0000)>>16;
			gsum+=(p & 0x00ff00)>>8;
			bsum+= p & 0x0000ff;
		    }
		    for (x=0;x<w;x++){
			a[yi]=dv[asum];
			r[yi]=dv[rsum];
			g[yi]=dv[gsum];
			b[yi]=dv[bsum];

			if(y==0){
			  vmin[x]=min(x+radius+1,wm);
			  vmax[x]=max(x-radius,0);
			} 
			p1=pix[yw+vmin[x]];
			p2=pix[yw+vmax[x]];
			
			asum+=((p1>>24) & 0xff)-((p2>>24) & 0xff); // asum
			rsum+=((p1 & 0xff0000)-(p2 & 0xff0000))>>16;
			gsum+=((p1 & 0x00ff00)-(p2 & 0x00ff00))>>8;
			bsum+= (p1 & 0x0000ff)-(p2 & 0x0000ff);
			yi++;
		    }
		    yw+=w;
		  }

		  for (x=0;x<w;x++){
		    asum=rsum=gsum=bsum=0;
		    yp=-radius*w;
		    for(i=-radius;i<=radius;i++){
			yi=max(0,yp)+x;
			asum+=a[yi]; // asum
			rsum+=r[yi];
			gsum+=g[yi];
			bsum+=b[yi];
			yp+=w;
		    }
		    yi=x;
		    for (y=0;y<h;y++){
			pix[yi] = (dv[asum]<<24) | (dv[rsum]<<16) | (dv[gsum]<<8) | dv[bsum]; 
			if(x==0){
			  vmin[y]=min(y+radius+1,hm)*w;
			  vmax[y]=max(y-radius,0)*w;
			} 
			p1=x+vmin[y];
			p2=x+vmax[y];
			
			asum+=a[p1]-a[p2]; // asum
			rsum+=r[p1]-r[p2];
			gsum+=g[p1]-g[p2];
			bsum+=b[p1]-b[p2];

			yi+=w;
		    }
		  }
		}