Final Project: Panopticon
The Panopticon is an installation that simulates a digital creature with a hundred-sided polygon of faces as a body. When this creature is looked at by the viewer, their face is stolen and added to the collective ball. The creature then uses the faces it has stolen to see in all directions, acting as the center of a massive cosmic panopticon. The database of faces is never purged: you cannot remove yourself from the sphere once you are a part of it.
With this project, I wanted to try to take the unseen and secret processes of modern surveillance and have them manifested as a mythical digital object/creature.
This project is still a work in progress, as there are still some features I want to get working. I wasn’t able to finish the system for creating the 3D face models from the Kinect in real-time, but this is what the ball will look like once I can get that system running with multiple faces (it only works with a single face in real-time right now):
Sketches:
Debugging the Face Stealer and more:
#pragma once
#include "ofMain.h"
#include "ofxFaceTracker.h"
#include "ofxAssimpModelLoader.h"
#include "ofxOpenCv.h"
#include "ofxKinect.h"
class PanopFace {
public:
ofVec3f position;
ofImage color,depth;
ofMesh mesh;
int age;
PanopFace() {
}
void setup() {
position.set(ofRandomf(),
ofRandomf(),
ofRandomf());
age = 0;
}
void fixToSphere() {
position.normalize();
}
};
class testApp : public ofBaseApp {
public:
void setup();
void updatePanopticon();
void updateFacestealer();
void update();
void drawPanopticon();
void drawFacestealer();
void draw();
void resetPanopticon();
void captureFace();
void keyPressed(int key);
// facestealer stuff
ofxKinect kinect;
ofxFaceTracker tracker;
int cropX;
int cropY;
int cropW;
int cropH;
bool faceCaptured;
int captureFaceTimer;
ofxCvColorImage kinectColor;
ofxCvGrayscaleImage kinectDepth;
ofImage capturedFaceColor;
ofImage capturedFaceDepth;
bool drawFacestealerDebug;
const float HTW_RATIO = 1.4;
const float SCALE_TO_PIXELS = 55.0;
const float Y_FUDGE_FACTOR = 7.0;
const float ORI_TOL_X = 0.2;
const float ORI_TOL_Y = 0.05;
const float ORI_TOL_Z = 0.15;
// panopticon stuff
int panopSize = 0;
float panopRadius = 0;
const float PANOP_RADIUS = 750.0;
const bool USE_KINECT_DEPTH = false;
ofEasyCam cam;
ofShader facesShader;
ofxAssimpModelLoader faceModel;
ofMesh generatedMesh;
ofxAssimpModelLoader starsMesh;
ofImage starsTexture;
ofMesh mesh;
vector faces;
};
#include "ofApp.h"
using namespace ofxCv;
using namespace cv;
void testApp::setup() {
ofSetVerticalSync(true);
ofDisableArbTex();
kinect.setRegistration(true);
kinect.init();
kinect.open();
kinect.setDepthClipping(500.0f, 1000.0f);
kinect.setCameraTiltAngle(10);
kinectColor.allocate(kinect.width, kinect.height);
kinectDepth.allocate(kinect.width, kinect.height);
tracker.setup();
tracker.setRescale(.5);
faceCaptured = false;
drawFacestealerDebug = false;
captureFaceTimer = 0;
if(USE_KINECT_DEPTH) {
faceModel.loadModel("models/plane.obj");
} else {
faceModel.loadModel("models/testsphere2.obj");
}
starsMesh.loadModel("models/stars.obj");
facesShader.load("shaders/faces.vert", "shaders/faces.frag");
starsTexture.loadImage("stars.gif");
cam.setupPerspective();
cam.setFov(10);
cam.setFarClip(100000);
resetPanopticon();
for(int i = 0; i < 10; i++) {
panopSize++;
faces.resize(panopSize);
faces[panopSize-1].setup();
faces[panopSize-1].color.loadImage("tempcolor.png");
faces[panopSize-1].depth.loadImage("tempdepth.png");
faces[panopSize-1].mesh = faceModel.getMesh(0);
}
}
void testApp::update() {
kinect.update();
// there is a new frame and we are connected
if(kinect.isFrameNew()) {
kinectDepth.setFromPixels(kinect.getDepthPixels(), kinect.width, kinect.height);
kinectDepth.flagImageChanged();
kinectColor.setFromPixels(kinect.getPixels(), kinect.width, kinect.height);
kinectColor.flagImageChanged();
}
updateFacestealer();
updatePanopticon();
}
void testApp::draw() {
ofBackground(0,0,0);
if(drawFacestealerDebug) {
drawFacestealer();
}
drawPanopticon();
}
void testApp::updateFacestealer() {
tracker.update(toCv(kinectColor));
cropX = tracker.getPosition().x;
cropY = tracker.getPosition().y;
float faceSize = tracker.getScale();
cropW = faceSize*SCALE_TO_PIXELS;
cropH = faceSize*SCALE_TO_PIXELS*HTW_RATIO;
cropY -= cropH/Y_FUDGE_FACTOR; // center of face was too low
captureFaceTimer++;
float orientationX = tracker.getOrientation().x;
float orientationY = tracker.getOrientation().y;
float orientationZ = tracker.getOrientation().z;
if(!(orientationX==0&&orientationY==0&&orientationZ==0) &&
abs(orientationX) < ORI_TOL_X &&
abs(orientationY) < ORI_TOL_Y &&
abs(orientationZ) < ORI_TOL_Z &&
captureFaceTimer > 30) {
captureFace();
captureFaceTimer = 0;
}
}
void testApp::drawFacestealer() {
ofDisableDepthTest();
ofSetColor(255, 255, 255);
kinectColor.draw(0, 0);
kinectDepth.draw(500,0);
if(faceCaptured) {
ofSetColor(255);
capturedFaceColor.draw(500, 500);
capturedFaceDepth.draw(700, 500);
ofSetColor(0, 255, 0, 150);
ofDrawBitmapString("extracted face texture", 0, 15);
}
ofSetColor(255, 0, 0, 150);
ofRect(cropX-cropW/2.0,
cropY-cropH/2.0,
cropW,
cropH);
ofSetColor(255);
tracker.draw();
ofSetColor(0, 0, 0, 255);
ofRect(0, 0, 230, 75);
ofSetColor(255, 100, 0, 150);
ofDrawBitmapString("looking for stealable face", 0, 15);
float orientationX = tracker.getOrientation().x;
float orientationY = tracker.getOrientation().y;
float orientationZ = tracker.getOrientation().z;
if(abs(orientationX) < ORI_TOL_X) {
ofSetColor(0, 255, 0, 150);
} else {
ofSetColor(255, 0, 0, 150);
}
ofDrawBitmapString("orientationX: " + ofToString(orientationX), 0, 30);
if(abs(orientationY) < ORI_TOL_Y) {
ofSetColor(0, 255, 0, 150);
} else {
ofSetColor(255, 0, 0, 150);
}
ofDrawBitmapString("orientationY: " + ofToString(orientationY), 0, 45);
if(abs(orientationZ) < ORI_TOL_Z) {
ofSetColor(0, 255, 0, 150);
} else {
ofSetColor(255, 0, 0, 150);
}
ofDrawBitmapString("orientationZ: " + ofToString(orientationZ), 0, 60);
}
void testApp::updatePanopticon() {
for(int i = 0; i < panopSize; i++) {
for(int j = 0; j < panopSize; j++) {
if(i != j) {
ofVec3f d = (faces[i].position.operator-(faces[j].position));
if(d.length() < 40.0 / panopSize) {
faces[i].position.operator+=(d.normalize().operator*=(0.01));
}
}
}
faces[i].fixToSphere();
if(faces[i].age < 255) faces[i].age+=10;
}
}
void testApp::drawPanopticon() {
ofEnableDepthTest();
glEnable(GL_CULL_FACE);
glCullFace(GL_FRONT);
cam.begin();
ofPushMatrix();
ofRotate(ofGetElapsedTimeMillis()*0.004, 0, 1, 0);
//facesShader.begin();
// 'heightmap' shader disabled because openframeworks is a strange animal
// about texture ids...could't get the damn thing to pass the right texture
// in...i know it's possible though because i got actually got it to work
// with only one texture (it's in the other project...go see it it's glorious)
for(int i = 0; i < panopSize; i++) {
float panopRadiusGoal = sqrt(panopSize/(4*PI))*16.0;
panopRadius += (panopRadiusGoal-panopRadius)*0.05;
float yrot = ofRadToDeg(atan(faces[i].position.x / faces[i].position.z));
if(faces[i].position.z < 0) yrot += 180;
yrot += 90;
ofPushMatrix();
ofTranslate(faces[i].position.x*-panopRadius,
faces[i].position.y*-panopRadius,
faces[i].position.z*-panopRadius);
ofRotate(yrot, 0, 1, 0);
ofRotate(-faces[i].position.y*90.0, 0, 0, 1);
ofScale(10, 10, 10);
// yeah same disabled texture uniform stuff for that god forsaken shader
/*
faces[i].depth.bind();
facesShader.setUniformTexture("tex0", faces[i].depth, 1);
ofTexture a = faces[i].color.getTextureReference();
ofLogNotice() << a.getTextureData().textureID;
facesShader.setUniformTexture("tex1",
faces[i].color,
a.getTextureData().textureID);
*/
ofSetColor(255,255,255, faces[i].age);
faces[i].color.bind();
faces[i].mesh.drawFaces();
ofPopMatrix();
}
ofPushMatrix();
ofScale(30,30,30);
starsTexture.bind();
ofSetColor(255,255,255,255);
starsMesh.getMesh(0).draw();
ofPopMatrix();
ofPopMatrix();
cam.end();
//facesShader.end();
}
void testApp::resetPanopticon() {
if(panopSize > 0) {
faces.resize(panopSize);
faces[panopSize-1].setup();
}
}
void testApp::captureFace() {
capturedFaceColor.setFromPixels(kinectColor.getPixels(),
kinect.getWidth(),
kinect.getHeight(),
OF_IMAGE_COLOR);
capturedFaceDepth.setFromPixels(kinectDepth.getPixels(),
kinect.getWidth(),
kinect.getHeight(),
OF_IMAGE_GRAYSCALE);
capturedFaceColor.crop(cropX-cropW/2.0,
cropY-cropH/2.0,
cropW,
cropH);
capturedFaceDepth.crop(cropX-cropW/2.0,
cropY-cropH/2.0,
cropW,
cropH);
// later this will save every face catpured so once your
// face is in the system, it stays forever !!
capturedFaceColor.saveImage("tempcolor.png");
capturedFaceDepth.saveImage("tempdepth.png");
panopSize++;
faces.resize(panopSize);
faces[panopSize-1].setup();
faces[panopSize-1].color.loadImage("tempcolor.png");
faces[panopSize-1].depth.loadImage("tempdepth.png");
generatedMesh = faceModel.getMesh(0);
if(USE_KINECT_DEPTH) {
for(int i = 0; i < generatedMesh.getVertices().size(); i++) {
ofVec3f v = generatedMesh.getVertex(i);
ofImage c = faces[panopSize-1].depth;
float rx = (v.z+1.0)/2.0;
float ry = (v.y+1.0)/2.0;
int x = (int)(rx*c.getWidth());
int y = (int)(ry*c.getHeight());
v.x = (c.getColor(x,y).r/255.0)*10.0;
generatedMesh.setVertex(i,v);
}
}
faces[panopSize-1].mesh = generatedMesh;
faceCaptured = true;
tracker.reset();
}
void testApp::keyPressed(int key) {
// capture face
if(key == 'c') {
captureFace();
}
// set to draw panopticon
if(key == 'p') {
resetPanopticon();
drawFacestealerDebug = false;
}
// set to draw facestealer debug
if(key == 's') {
drawFacestealerDebug = true;
}
// randomize face positions (if they get stuck)
if(key == 'r') {
for(int i = 0; i < panopSize; i++) {
faces[i].setup();
faces[i].fixToSphere();
}
}
}