zbeok-Mocap

DESCRIPTION:

As an enthusiast of 3D animation and the technology involved, I wanted to experiment with what was beyond what I already knew. Translated to this project, this meant doing something with a little bit of complexity on the math side. I took much inspiration from William Forsythe’s videos because such a mathematical analysis of dance was satisfying to me, as someone who is bad at dancing and had only been forced into ballet as an ungainly child. The result is an algorithm that takes the positions of joints and tries to create lines using a spline curve to interpolate the in-between points. The split between lines is mainly based on their velocities. This project was intended to be realtime, but seeing just the prototype works as well.

Overall I’m pretty satisfied with the idea, but the execution could definitely be polished. Its presentation is very “design by engineer” which as an art major, kind of offends me.

SKETCHES:

IMAGES:

This take was a bvh file of Golan stumbling around.

This take is from a Mixamo FBX file of a dancing model.

GIFS:

VIDEOS:

CODE:

if ( ! Detector.webgl ) Detector.addGetWebGLMessage();

var container, controls;
var camera, scene, renderer, light;
var mixer,skeletonHelper;
var clock = new THREE.Clock();

var mixers = [];
var skelly;
var joints = [];
var jointPos =[];
var pastJointPos =[];
var jointVels=[];
var trails = [];
var trailViable = [];
var trailOn = [];

var splineHelperObjects = [];
var splinePositions = [];
var splineGeos = [];

var ARC_SEGMENTS = 200;
var splineMeshes=[];

var splines = [];

init();
animate();

function init() {

	container = document.createElement( 'div' );
	document.body.appendChild( container );

	camera = new THREE.PerspectiveCamera( 45, window.innerWidth / window.innerHeight, 1, 2000 );
	camera.position.set( 100, 200, 300 );

	controls = new THREE.OrbitControls( camera );
	controls.target.set( 0, 100, 0 );
	controls.update();

	scene = new THREE.Scene();
	scene.background = new THREE.Color( 0x000);
	// scene.fog = new THREE.Fog( 0xa0a0a0, 200, 1000 );

	light = new THREE.HemisphereLight( 0xffffff, 0x444444 );
	light.position.set( 0, 200, 0 );
	scene.add( light );

	light = new THREE.DirectionalLight( 0xffffff );
	light.position.set( 0, 200, 100 );
	light.castShadow = true;
	light.shadow.camera.top = 180;
	light.shadow.camera.bottom = -100;
	light.shadow.camera.left = -120;
	light.shadow.camera.right = 120;
	scene.add( light );


	var grid = new THREE.GridHelper( 2000, 20, 0xa0a0a0, 0xa0a0a0 );
	grid.material.opacity = .2;
	grid.material.transparent = true;
	scene.add( grid );

	// model

	var loader = new THREE.BVHLoader();
	loader.load("stumble.bvh", function( result ) {
		skeletonHelper = new THREE.SkeletonHelper( result.skeleton.bones[ 0 ] );
		skeletonHelper.skeleton = result.skeleton; // allow animation mixer to bind to SkeletonHelper directly
		console.log(result.skeleton);
		var boneContainer = new THREE.Group();
		boneContainer.add( result.skeleton.bones[ 0 ] );
		scene.add( skeletonHelper );

		scene.add( boneContainer );

		// play animation
		mixer = new THREE.AnimationMixer( skeletonHelper );
		mixer.clipAction( result.clip ).setEffectiveWeight( 1.0 ).play();
		console.log(mixer);
		skelly= result.skeleton;
		jointSet();
		// scene.add( result );

	} );

	renderer = new THREE.WebGLRenderer();
	renderer.setPixelRatio( window.devicePixelRatio );
	renderer.setSize( window.innerWidth, window.innerHeight );
	renderer.shadowMap.enabled = true;
	container.appendChild( renderer.domElement );

	window.addEventListener( 'resize', onWindowResize, false );

}

function onWindowResize() {

	camera.aspect = window.innerWidth / window.innerHeight;
	camera.updateProjectionMatrix();

	renderer.setSize( window.innerWidth, window.innerHeight );

}

//

function animate() {

	requestAnimationFrame( animate );

	var delta = clock.getDelta();
	if ( mixer ) mixer.update( delta );
	if (skelly) {
		jointUpdate();
		jointVelProcess();
	}
	renderer.render( scene, camera );
}

function jointSet() {
	for (var i=0;i<skelly.bones.length;i++) {
		var geometry = new THREE.SphereGeometry( 1, 1, 1 );
		var material = new THREE.MeshBasicMaterial( {color: 0xffff00} );
		var sphere = new THREE.Mesh( geometry, material );
		var curr_bone_pos=skelly.bones[i].getWorldPosition();
		sphere.position.x=curr_bone_pos.x;
		sphere.position.y=curr_bone_pos.y;
		sphere.position.z=curr_bone_pos.z;
		scene.add( sphere );
		joints.push(sphere);
		jointPos.push(curr_bone_pos);
		pastJointPos = jointPos.slice();
		jointVels.push(new THREE.Vector3(0,0,0));
		trailViable.push(false);
		var arr=[];
		splineHelperObjects.push(arr);
		splinePositions.push(arr);
		splineGeos.push(arr);
		var spline = {};
		splines.push(spline);
		trailOn.push(false);
	}
	
}

function jointUpdate() {
	pastJointPos = jointPos.slice();
	for (var i=0;i<joints.length;i++) {
		var curr_bone_pos=skelly.bones[i].getWorldPosition();
		joints[i].position.x=(curr_bone_pos.x);
		joints[i].position.y=(curr_bone_pos.y);
		joints[i].position.z=(curr_bone_pos.z);

		jointPos[i]=(curr_bone_pos);
	}
}

function getNorm(v) {
	return (Math.pow((v.x)*(v.x)+(v.y)*(v.y)+(v.z)*(v.z),.5));
}
function jointVelProcess() {
	for (var i=0;i<joints.length;i++) { jointVels[i]=getNorm(jointPos[i].sub(pastJointPos[i])); jointVis(jointVels[i],skelly.bones[i],i); } var j=11; // jointVis(jointVels[j],skelly.bones[j],j); } function jointVis(vel,curr_bone,i) { var curr_bone_pos=curr_bone.getWorldPosition(); if (vel>230)  {
		if (!(trailOn[i]) && splinePositions[i].length>=2) {
			initTrail(i);
			trailOn[i]=true;
		}
		trailViable[i]=false;
	}
	if (vel<150 && trailViable[i]==false){
		splinePositions[i].push(curr_bone_pos);
		if (trailOn[i])  {
			addPoint(i,curr_bone_pos);
			if (i==11) updateSplineOutline(i);
		}
		trailViable[i]=true;
		
	}
}
function initTrail(i) {
	trailOn[i]=true;
	// console.log("initTrail");
	splineInit(i);
}


function addPoint(i,pos) {
	splinePositions[i].push( pos );
	updateSplineOutline(i);

}

function updateSplineOutline(i) {

	var spline = splines[i].centripetal;
	splineMeshes[i] = spline.mesh;
	// var initSp = console.log(spline.getPoint( t, p ));

	for ( var j = 0; j < ARC_SEGMENTS; j ++ ) {
		var p = splineMeshes[i].geometry.vertices[ j ];
		var t = j /  ( ARC_SEGMENTS - 1 );
		spline.getPoint( t, p );
		// if (initSp!=spline.getPoint(t,p) ) console.log(spline.getPoint( t, p ));

	}
	splineMeshes[i].geometry.verticesNeedUpdate = true;



}




function addSplineObject( position,i ) {

	var material = new THREE.MeshLambertMaterial( { color: Math.random() * 0xffffff } );
	var object = new THREE.Mesh( splineGeos[i], material );

	object.position.copy( position );

	// scene.add( object );
	// splineHelperObjects[i].push( object );
	// return object;

}

function splineInit(i) {
	// splinePositions[i].push( splineHelperObjects[i][0].position );
	var geometry = new THREE.Geometry();
	for ( var j = 0; j < ARC_SEGMENTS; j ++ ) {
		geometry.vertices.push( new THREE.Vector3() );
	}
	splineGeos[i]=geometry;

	var curve = new THREE.CatmullRomCurve3( splinePositions[i]);
	curve.curveType = 'centripetal';
	curve.mesh = new THREE.Line( geometry.clone(), new THREE.LineBasicMaterial( {
		color: 0xffffff,
		opacity: .35,
		linewidth: 3
		} ) );
	trails.push(curve);
	splines[i].centripetal = curve;

	var spline = splines[ i ].centripetal;
	scene.add( spline.mesh );
	setTimeout(function(){ removeSpline(spline.mesh,splinePositions[i],i); }, 500);
}

function removeSpline(curve,pts,i) {
	
	while ( (pts.length) <= 0 ) {
		pts.pop(0);
	}
	scene.remove(curve); 
	splinePositions[i]=[];
	trailOn[i]=false;
}