faitk – Mocap

Moments of Brutal Kill

For this project I wanted to use Open Pose to do some experiments. In the beginning I had a hard time deciding what kinds of videos I wanted to extract data from. I decided to use a compilation of “brutal kill moments” from GTA . It was honestly quite shocking to see some of the brutal footage. Because even though we know that what is happening on the screen isn’t real, those actions can technically happen. It’s also odd to think that people are controlling this environment and doing these brutal things.

I thought it’d be interesting to extract such grotesque footage of not “real people” but actions inhibited by the gamers. Conceptually, I wanted to explore the violent nature of humans and the visceral reactions whilst viewing them. I decided to map Kim Jung Un’s face onto the extracted motion data. In this short moving picture Kim Jung Un is repeatedly wreaking havoc and violence to himself (and others). By mapping the sound from the original footage there is a little more context and environment that people can imagine.

Raw Footage

Thanks to Golan and Claire for always being around and helping in every way.

Thank you to David for compiling open pose so that I could use it.

Thank you to Aman and Marissa for always assisting me in my code.

PImage img; 
PImage img2;
JSONArray baseJSON;
SceneSequence sequence;
import processing.video.*;
Movie myMovie;
 
void setup()
{
  size(1280, 720);
  myMovie = new Movie(this, "flagwaving.mp4");
  myMovie.loop();
 
  //size(1280, 720);
  stroke(255);
  strokeWeight(10);
  img = loadImage("kim_small.png");
  //img2 = loadImage("flag.jpg");
  baseJSON = loadJSONArray("output2.json");
  sequence = new SceneSequence(baseJSON); 
}
 
void movieEvent(Movie m) {
  m.read();
}
 
 
int x = 0;
 
 
void draw()
{
  image(myMovie, width/2, height/2);
  //image(img2,0,0);
  //image(img2,width/2,height/2);
  //background(0);
  sequence.drawFrame();
  fill(255); 
  text(frameCount, mouseX, mouseY); 
 
  if (x < 1950) {
    //line(x, 0, x, 1000);
    x = x + 1;
  } else {
    noLoop();
  }
  // Saves each frame as screen-0001.tif, screen-0002.tif, etc.
  saveFrame("line-######.jpg"); 
}
 
 
 
 class SceneSequence //aka frames
{
  int sceneIndex = 0;
  int numScenes;
  int imageWidth, imageHeight;
  String poseModel;
  JSONObject[][] poses; //for each frame, there is an array of bodies
 
  public SceneSequence(JSONArray baseJSON)
  {
 
    JSONObject baseObj = baseJSON.getJSONObject(0);
    numScenes = baseObj.getInt("numScenes");
    poses = new JSONObject[numScenes][];
    imageWidth = baseObj.getInt("imageWidth");
    imageHeight = baseObj.getInt("imageHeight");
    poseModel = baseObj.getString("poseModel");
    for(int sceneIndex = 0; sceneIndex < numScenes; sceneIndex++)
    {
      JSONObject scene = baseObj.getJSONObject("" + sceneIndex);
      int numPoses = scene.getInt("numPoses");
      poses[sceneIndex] = new JSONObject[numPoses];
      for(int poseIndex = 0; poseIndex < numPoses; poseIndex++)
      {
        JSONObject pose = scene.getJSONObject("" + poseIndex);
        poses[sceneIndex][poseIndex] = pose;
      }
    }
  }
  public void drawFrame()
  {
 
    JSONObject[] scene = poses[sceneIndex];
    for(int i = 0; i < scene.length; i++) { JSONObject pose = scene[i]; //println(poseModel); if(poseModel.equals("COCO")) drawCOCOPose(pose); else drawMPIPose(pose); } sceneIndex++; if(sceneIndex >= numScenes)
      sceneIndex = 0;
  }
 
}
 
float headSize(JSONObject pose) {
  JSONArray kneePose = pose.getJSONArray("right_knee");
  float kneeX = kneePose.getFloat(0);
  float kneeY = kneePose.getFloat(1);
 
  JSONArray neckPose = pose.getJSONArray("neck");
  float neckX = neckPose.getFloat(0);
  float neckY = neckPose.getFloat(1);
 
  if(kneeX == 0 || kneeY ==0 || neckX == 0 || neckY==0){
    return 50;
  }
 
 
  float bodyHeight = sqrt(sq(kneeX-neckX) + sq(kneeY-neckY)) ;
 
   return bodyHeight; //must *return value* to use bc it is returning a number, just computing a number 
 
}
 
//draw a line if its 0
void connectJoints(JSONObject pose, String joint0, String joint1)
{
  //println("joint0 is?" + joint0);
  JSONArray pt0 = pose.getJSONArray(joint0);
  JSONArray pt1 = pose.getJSONArray(joint1);
  float x0 = pt0.getFloat(0);
  float y0 = pt0.getFloat(1);
  float x1 = pt1.getFloat(0);
  float y1 = pt1.getFloat(1);   
 
  if((x0 != 0) && (y0 != 0) && (x1 != 0) && (y1 != 0)){
 
     if(joint0 =="nose"){
    //float m = map(drawHead(x0,y0), imageWidth, imageHeight, x0, y0);  
      drawHead(x0, y0, headSize(pose)*.5);
    }else{
      line(x0, y0, x1, y1);
    }
 
  }
  stroke(255, 231, 201);
 
}
 
void drawMPIPose(JSONObject pose)
{
  connectJoints(pose, "head", "neck");
  connectJoints(pose, "neck", "left_shoulder");
  connectJoints(pose, "neck", "right_shoulder");
  connectJoints(pose, "neck", "left_hip");
  connectJoints(pose, "neck", "right_hip");
  connectJoints(pose, "left_shoulder", "left_elbow");
  connectJoints(pose, "left_elbow", "left_hand");
  connectJoints(pose, "right_shoulder", "right_elbow");
  connectJoints(pose, "right_elbow", "right_hand");
  connectJoints(pose, "left_hip", "left_knee");
  connectJoints(pose, "left_knee", "left_foot");
  connectJoints(pose, "right_hip", "right_knee");
  connectJoints(pose, "right_knee", "right_foot");
}
 
void drawCOCOPose(JSONObject pose)
{
  connectJoints(pose, "left_eye", "nose");
  connectJoints(pose, "right_eye", "nose");
  connectJoints(pose, "left_ear", "left_eye");
  connectJoints(pose, "right_ear", "right_eye");
 
  connectJoints(pose, "neck", "left_shoulder");
  connectJoints(pose, "left_shoulder", "left_elbow");
  connectJoints(pose, "left_elbow", "left_wrist");
  connectJoints(pose, "neck", "right_shoulder");
  connectJoints(pose, "right_shoulder", "right_elbow");
  connectJoints(pose, "right_elbow", "right_wrist");
  connectJoints(pose, "neck", "left_hip");
  connectJoints(pose, "left_hip", "left_knee");
  connectJoints(pose, "left_knee", "left_foot");
  connectJoints(pose, "neck", "right_hip");
  connectJoints(pose, "right_hip", "right_knee");
  connectJoints(pose, "right_knee", "right_foot");
  connectJoints(pose, "nose", "neck");
}
 
void drawHead(float x, float y, float w)
{
 println("headwidth" + w );
  //need to map the width of head to the value of the changing x0s and x1s?
  //image(loadImage("kim_small.png"), x0, x1, m, 50);
   //float headw = constrain(w,10,160);
  image(loadImage("kim_small.png"), x, y, w, w*1.5);
  imageMode(CENTER);
}