aahdee-Mocap

Man this one was a rollercoaster.

First are my sketches.

My first concept was to have a field of repeating shapes where a body can be formed by turning the shapes into a different shape. The body would move around and thus alter the field. I turned away from that idea since it didnt seem interesting. I then wanted to think of the interactions and connections between two people, so I decided to look into the movements of ballroom dancers. I’m not comfortable with dancing and I felt as if professional dancers would be able show strong synergy with their partners, so I decided to use OpenPose to grab their motions for this project.

I found this video, which I really wanted to use it since the dancing was very elegant, but OpenPose had a difficult time choosing the focus points on the woman because of her dress, so I opted for a clip from this video.

Now, here is my final project:

I spent most of my time on this project getting OpenPose to work optimally for me and working around it’s limitations. When I produced the results, I was satisfied with just the wireframes, but when I launched Premiere Pro to sync the music with the result, I thought that the wireframe layed over the source material looked more pleasing and ensured that the viewer saw the wireframes as a pair of ballroom dancers. Overall, I’m surprisingly satisfied with this result. I thought that this was going to be another difficult week but it was really pleasing.

Finally, my code.

 

//a lot of this was given to be me by Golan. 
//this turns a JSON file into the wireframe model. 
String[] parts = {"left_elbow","left_foot","left_hip","left_knee","left_shoulder","left_wrist",
                "neck","right_elbow","right_foot","right_hip","right_knee","right_shoulder","right_wrist"};
 
class SceneSequence
{
  int sceneIndex = 0;
  int numScenes;
  int imageWidth, imageHeight;
  String poseModel;
  JSONObject[][] poses;
  public SceneSequence(JSONArray baseJSON)
  {
    JSONObject baseObj = baseJSON.getJSONObject(0);
    numScenes = baseObj.getInt("numScenes");
    poses = new JSONObject[numScenes][];
    imageWidth = baseObj.getInt("imageWidth");
    imageHeight = baseObj.getInt("imageHeight");
    poseModel = baseObj.getString("poseModel");
    for(int sceneIndex = 0; sceneIndex < numScenes; sceneIndex++)
    {
      JSONObject scene = baseObj.getJSONObject("" + sceneIndex);
      int numPoses = scene.getInt("numPoses");
      poses[sceneIndex] = new JSONObject[numPoses];
      for(int poseIndex = 0; poseIndex < numPoses; poseIndex++)
      {
        JSONObject pose = scene.getJSONObject("" + poseIndex);
        poses[sceneIndex][poseIndex] = pose;
      }
    }
  }
  public void drawFrame()
  {
    JSONObject[] scene = poses[sceneIndex];
    for(int i = 0; i < scene.length; i++) { JSONObject pose = scene[i]; println(poseModel); if(poseModel.equals("COCO")) drawCOCOPose(pose); else drawMPIPose(pose); } sceneIndex++; if(sceneIndex >= numScenes)
      sceneIndex = 0;
  }
}
 
void connectJoints(JSONObject pose, String joint0, String joint1)
{
  JSONArray pt0 = pose.getJSONArray(joint0);
  JSONArray pt1 = pose.getJSONArray(joint1);
  float x0 = pt0.getFloat(0);
  float y0 = pt0.getFloat(1);
  float x1 = pt1.getFloat(0);
  float y1 = pt1.getFloat(1);
  if((x0 != 0) && (y0 != 0) && (x1 != 0) && (y1 != 0))
    line(x0, y0, x1, y1);
}
 
void drawMPIPose(JSONObject pose)
{
  connectJoints(pose, "head", "neck");
  connectJoints(pose, "neck", "left_shoulder");
  connectJoints(pose, "neck", "right_shoulder");
  connectJoints(pose, "neck", "left_hip");
  connectJoints(pose, "neck", "right_hip");
  connectJoints(pose, "left_shoulder", "left_elbow");
  connectJoints(pose, "left_elbow", "left_hand");
  connectJoints(pose, "right_shoulder", "right_elbow");
  connectJoints(pose, "right_elbow", "right_hand");
  connectJoints(pose, "left_hip", "left_knee");
  connectJoints(pose, "left_knee", "left_foot");
  connectJoints(pose, "right_hip", "right_knee");
  connectJoints(pose, "right_knee", "right_foot");
}
 
void drawCOCOPose(JSONObject pose)
{
  connectJoints(pose, "left_eye", "nose");
  connectJoints(pose, "left_eye", "neck");
  connectJoints(pose, "right_eye", "neck");
  connectJoints(pose, "right_eye", "nose");
  connectJoints(pose, "left_ear", "left_eye");
  connectJoints(pose, "right_ear", "right_eye");
  connectJoints(pose, "left_ear", "neck");
  connectJoints(pose, "right_ear", "neck");
  connectJoints(pose, "left_ear", "right_ear");
  connectJoints(pose, "nose", "neck");
  connectJoints(pose, "neck", "left_shoulder");
  connectJoints(pose, "left_shoulder", "left_elbow");
  connectJoints(pose, "left_shoulder", "left_hip");
  connectJoints(pose, "left_elbow", "left_wrist");
  connectJoints(pose, "neck", "right_shoulder");
  connectJoints(pose, "right_shoulder", "right_elbow");
  connectJoints(pose, "right_shoulder", "right_hip");
  connectJoints(pose, "right_elbow", "right_wrist");
  connectJoints(pose, "right_hip", "left_hip");
  connectJoints(pose, "left_hip", "left_knee");
  connectJoints(pose, "left_knee", "left_foot");
  connectJoints(pose, "right_hip", "right_knee");
  connectJoints(pose, "right_knee", "right_foot");
}
 
 
 
JSONArray baseJSON;
SceneSequence sequence;
 
void connectBodies()
{
  int currentSceneIndex = sequence.sceneIndex;
  JSONObject[][] allScenes = sequence.poses;
  JSONObject[] currentScene = allScenes[currentSceneIndex];
  if (currentScene.length > 1) {
    JSONObject pose0 = currentScene[0];
    JSONObject pose1 = currentScene[1];
    for (String part : parts)
    {
      JSONArray pt0 = pose0.getJSONArray(part);
      JSONArray pt1 = pose1.getJSONArray(part);
      if ((pt0 != null) && (pt1 != null) && (pt0.getFloat(0) != 0) && (pt1.getFloat(0) != 0)) {
        float x0 = pt0.getFloat(0);
        float y0 = pt0.getFloat(1);
        float x1 = pt1.getFloat(0);
        float y1 = pt1.getFloat(1);
        stroke(136, 0, 10); 
        strokeWeight(2); 
        line(x0, y0, x1, y1);
      }
    }
  }
}
 
void setup()
{
  size(640, 360);
  stroke(255);
  baseJSON = loadJSONArray("outputn.json");
  sequence = new SceneSequence(baseJSON);
  frameRate(15);
}
 
void draw()
{
  background(255, 117, 103);
  stroke(255); 
  strokeWeight (2);
  sequence.drawFrame();
  fill(255); 
  connectBodies();
 
}

Shout out to Golan and Claire, damn, wow, woo.