[work 119] DancingOnTheInside

Movie

Source code

about

• 星野源の「うちで踊ろう」動画をモザイク加工する
• 音に合わせて波の振幅が変化するようにする

file

• 上部にあるファイル名が表示されているボタンを押すと、表示されるファイルが切り替わります
• 別ウィンドウ表示したい時や行番号などが無いRawMode表示したい時は、コード内右上のボタンを押してください（ボタンはマウスオーバーすると表示されます）

#include "ofMain.h"
#include "ofApp.h"

//================================
int main( ){

    // 4K:4096x2160
    // 2K:2048x1080
    // FullHD:1920x1080
    // HD:1440x1080
    // HD720p:1280x720
    // DVD:720x480
    // setup the GL context
//    ofSetupOpenGL(1280,720, OF_WINDOW);
    ofSetupOpenGL(1024,1024, OF_WINDOW);

	// this kicks off the running of my app
	// can be OF_WINDOW or OF_FULLSCREEN
	// pass in width and height too:
	ofRunApp( new ofApp());

}

#pragma once

#include "ofMain.h"
#include "ofxGui.h"

#include "WavyLine.hpp"
#include "Lyrics.hpp"


class ofApp : public ofBaseApp{
public:
    ofApp();
    ~ofApp();
    
    void setup();
    void update();
    void draw();
    
    void keyPressed(int key);
    void keyReleased(int key);
    void mouseMoved(int x, int y);
    void mouseDragged(int x, int y, int button);
    void mousePressed(int x, int y, int button);
    void mouseReleased(int x, int y, int button);
    void mouseEntered(int x, int y);
    void mouseExited(int x, int y);
    void windowResized(int w, int h);
    void dragEvent(ofDragInfo dragInfo);
    void gotMessage(ofMessage msg);
    
private:
    ofVideoPlayer movie;
    unique_ptr<WavyLine> mosaic;
    Lyrics lyrics;
    
    ofSoundPlayer sound;
};

#include "ofApp.h"
#include <chrono>


ofApp::ofApp(){
    
}

ofApp::~ofApp(){
    
}

//--------------------------------------------------------------
void ofApp::setup(){
    double fps = 30;
    
    ofSetFrameRate(fps);
    ofBackground(ofColor::white);
    ofSetBackgroundAuto(true);
    ofSetVerticalSync(true);
    
    lyrics.setup();
    
    movie.load("input/DancingOnTheInsideVideo.mp4");
    movie.setLoopState(OF_LOOP_NONE);
    while(!movie.isLoaded()) {
        std::cout << "loding..." << std::endl;
    }
    movie.play();
    
    sound.load("input/DancingOnTheInsideAudio.mp3");
    sound.setLoop(false);
    sound.play();
    
    mosaic = make_unique<WavyLine>(20, 8);
    mosaic->setup();
}

//--------------------------------------------------------------
void ofApp::update(){
    auto t0 = std::chrono::high_resolution_clock::now();
    
    movie.update();
    ofSoundUpdate();
    
    if (movie.isFrameNew()) {
        lyrics.update(sound);
        
        ofPixels pix = movie.getPixels();
        pix.setImageType(OF_IMAGE_COLOR_ALPHA);
        mosaic->update(pix);
        mosaic->setAmpLimit(sound);
    }
    
    auto t1 = std::chrono::high_resolution_clock::now();
    cout << "update(): " << std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count() / 1000.0 << " sec" << endl;
}


//--------------------------------------------------------------
void ofApp::draw(){
    auto t0 = std::chrono::high_resolution_clock::now();
    
    if (mosaic != nullptr) {
        mosaic->display();
        lyrics.display();
    }
    
    std::cout << "fps: " << ofGetFrameRate() << std::endl;
    
    auto t1 = std::chrono::high_resolution_clock::now();
    cout << "draw(): " << std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count() / 1000.0 << " sec" << endl;
}

#ifndef Mosaic_hpp
#define Mosaic_hpp

#include <stdio.h>
#include "ofMain.h"

class Mosaic {
public:
    Mosaic(int c, int f);
    ~Mosaic();
    void setup();
    void update(ofPixels &p);
    virtual void display();
    
protected:
    int factor;
    int cellSize;
    int cols, rows;
    std::vector<ofColor> colors;
    std::vector<int> toneIdxes;
    
private:
    
};
#endif /* Mosaic_hpp */

#include "Mosaic.hpp"
#include <chrono>


Mosaic::Mosaic(int c, int f)
{
    std::cout << "Mosaic:Start" << std::endl;
    cellSize = c;
    factor = f - 1;
}


Mosaic::~Mosaic()
{
    std::cout << "Mosaic:End" << std::endl;
}


void Mosaic::setup()
{
    std::cout << "Mosaic:setup" << std::endl;
}


void Mosaic::update(ofPixels &p)
{
    std::cout << "Mosaic:update" << std::endl;
    
    ofPixels pixs = p;
    colors.clear();
    toneIdxes.clear();
    
    int cs = 0;
    if (pixs.getWidth() > pixs.getHeight()) {
        int h = (pixs.getWidth() * ofGetHeight()) / ofGetWidth();
        pixs.resize(pixs.getWidth(), h);
        cs = cellSize * pixs.getWidth() / ofGetWidth();
    } else {
        int w = (pixs.getHeight() * ofGetWidth()) / ofGetHeight();
        pixs.resize(w, pixs.getHeight());
        cs = cellSize * pixs.getHeight() / ofGetHeight();
    }
    
    cols = (int)(pixs.getWidth() / cs);
    rows = (int)(pixs.getHeight() / cs);
    
    for (int row = 0; row < rows; row++) {
        for (int col = 0; col < cols; col++) {
            int sum_r = 0;
            int sum_g = 0;
            int sum_b = 0;
            for (int j = 0; j < cs; j++) {
                for (int i = 0; i < cs; i++) {
                    ofColor c;
                    int x = col * cs + i;
                    int y = row * cs + j;
                    c = pixs.getColor(x, y);
                    sum_r += c.r;
                    sum_g += c.g;
                    sum_b += c.b;
                }
            }
            
            int ave_r = sum_r / (cs * cs);
            int ave_g = sum_g / (cs * cs);
            int ave_b = sum_b / (cs * cs);
            
            int tone_r = factor * ave_r / 255 * (255 / factor);
            int tone_g = factor * ave_g / 255 * (255 / factor);
            int tone_b = factor * ave_b / 255 * (255 / factor);
            ofColor tone = ofColor(tone_r, tone_g, tone_b);
            colors.push_back(tone);
            
            int index = factor * (int)tone.getBrightness() / 255;
            toneIdxes.push_back(index);
        }
    }
}


void Mosaic::display()
{
    std::cout << "Mosaic:display" << std::endl;
    
    for (int row = 0; row < rows; row++) {
        for (int col = 0; col < cols; col++) {
            int idx = cols * row + col;
            ofVec2f corner = ofVec2f(col * cellSize, row * cellSize);
            ofPushMatrix();
            ofSetRectMode(OF_RECTMODE_CORNER);
            ofTranslate(corner);
            ofFill();
            ofSetColor(colors[idx]);
            ofDrawRectangle(0, 0, cellSize, cellSize);
            ofPopMatrix();
        }
    }
}

#ifndef WavyLine_hpp
#define WavyLine_hpp

#include <stdio.h>
#include "ofMain.h"
#include "Mosaic.hpp"


struct WavePattern {
    float freq;
    float amp;
    
    WavePattern(float f, float a) : freq(f), amp(a) {};
};


struct WaveData {
    std::vector<ofVec2f> wavePoint;
    ofColor color;
    
    WaveData(){};
    WaveData(std::vector<ofVec2f> &p, ofColor c) : wavePoint(p), color(c) {};
};

class WavyLine : public Mosaic {
public:
    WavyLine(int c, int f);
    ~WavyLine();
    void setup();
    void update(ofPixels &p);
    void display();
    void setAmpLimit(ofSoundPlayer &sound);
    
private:
    
    std::vector<WavePattern> pattern;
    ofVec2f terminatePoint;
    float amplimit;
    float threshold;
    float oldLow;
    float limit;
    static constexpr size_t nBandsToGet = 128;
    std::array<float, nBandsToGet> fftSmoothed{{0}};
    
    std::vector<WaveData> waveList;
};
#endif /* WavyLine_hpp */

#include "WavyLine.hpp"
#include <chrono>



WavyLine::WavyLine(int c, int f) : Mosaic(c, f)
{
    std::cout << "WavyMosaic:Start" << std::endl;
}


WavyLine::~WavyLine()
{
    std::cout << "WavyMosaic:End" << std::endl;
}


void WavyLine::setup()
{
    std::cout << "WavyMosaic:setup" << std::endl;
    
    Mosaic::setup();
    
    float amp = ((cellSize / 2) * 0.8);
    pattern.push_back(WavePattern(5, amp));
    pattern.push_back(WavePattern(4, amp));
    pattern.push_back(WavePattern(3, amp));
    pattern.push_back(WavePattern(2, amp));
    pattern.push_back(WavePattern(1, amp));
    pattern.push_back(WavePattern(2, amp / 2));
    pattern.push_back(WavePattern(1, amp / 2));
    pattern.push_back(WavePattern(1, 0));
    
    amplimit = 1.0;
    threshold = 0.1;
    oldLow = 0;
    limit = 0;
    fftSmoothed.fill(0);
}


void WavyLine::update(ofPixels &p)
{
    std::cout << "WavyMosaic:update" << std::endl;
    
    Mosaic::update(p);
    
    waveList.clear();
    
    for (int r = 0; r < rows; r++) {
        int x = 0;
        for (int c = 0; c < cols; c++) {
            int idx = cols * r + c;
            int toneIdx = toneIdxes.at(idx);
            
            std::vector<ofVec2f> w;
            ofColor color = colors.at(idx);
            
            float step = TWO_PI / cellSize;
            for (float theta = 0; theta < TWO_PI; theta += step) {
                float t = pattern[toneIdx].freq * theta;
                float px = x++;
                float py = std::sin(t) * pattern[toneIdx].amp * amplimit + (cellSize * r + cellSize / 2);
                w.emplace_back(px, py);
            }
            w.emplace_back(x, (cellSize * r + cellSize / 2)); //terminate point
            
            waveList.emplace_back(w, color);
        }
    }
}


void WavyLine::display()
{
    std::cout << "WavyMosaic:display" << std::endl;
    
    for (WaveData &wd : waveList) {
        ofPushMatrix();
        ofPushStyle();
        ofNoFill();
        ofBeginShape();
        ofSetColor(wd.color);
        ofVertices(wd.wavePoint);
        ofEndShape();
        ofPopStyle();
        ofPopMatrix();
    }
}


void WavyLine::setAmpLimit(ofSoundPlayer &sound)
{
    if (sound.isPlaying()) {
        float *val = ofSoundGetSpectrum(nBandsToGet);
        
        for (int i = 0;i < nBandsToGet; i++){
            fftSmoothed[i] *= 0.96f;

            if (fftSmoothed[i] < val[i]) {
                fftSmoothed[i] = val[i];
            }
        }
        
        float low = 0;
        int num = 2;
        for (int i = 0; i < num; i++) {
            low += fftSmoothed[i];
        }
        low /= num;
        
        if (low - oldLow > threshold) {
            limit = 1.0;
        } else {
            limit *= 0.95;
        }
        oldLow = low;
        
        amplimit = ofMap(limit, 0, 1, 0.3, 2.5);
        std::cout << "amplimit: " << amplimit << std::endl;
    } else {
        std::cout << "sound not playing" << std::endl;
    }
}

#ifndef Lyrics_hpp
#define Lyrics_hpp

#include <stdio.h>
#include <chrono>
#include "ofMain.h"


struct LyricSet {
    std::string lyric;
    ofColor color;
    int startCount;
    int period;
    
    LyricSet(std::string l, ofColor c, int cnt, int p) : lyric(l), color(c), startCount(cnt), period(p) {};
};


class Lyrics {
public:
    Lyrics();
    ~Lyrics();
    void setup();
    void update(ofSoundPlayer &sound);
    void display();
    
private:
    bool start;
    ofTrueTypeFont font;
    std::vector<LyricSet> lyrics;
    
    ofColor base;
    ofColor text;
    
    std::chrono::high_resolution_clock::time_point startTime;
};
#endif /* Lyrics_hpp */

#include "Lyrics.hpp"


Lyrics::Lyrics()
{
    
}


Lyrics::~Lyrics()
{
    
}


void Lyrics::setup()
{
    ofTrueTypeFontSettings settings("font/HiraginoProNW4.ttc", 32);
    settings.addRange(ofUnicode::Latin);
    settings.addRanges(ofAlphabet::Japanese);
    settings.contours = true;
    font.load(settings);

    base = ofColor(ofColor::white);
    text = ofColor(32,83,143);
    
    float fps = ofGetTargetFrameRate();
    lyrics.push_back(LyricSet("たまに重なり合うよな　僕ら", ofColor(text), 0, 7));
    lyrics.push_back(LyricSet("扉閉じれば　明日が生まれるなら", ofColor(text), 8, 4));
    lyrics.push_back(LyricSet("遊ぼう　一緒に", ofColor(text), 13, 5));
    
    lyrics.push_back(LyricSet("うちで踊ろう　ひとり踊ろう", ofColor(text), 20, 4));
    lyrics.push_back(LyricSet("変わらぬ鼓動　弾ませろよ", ofColor(text), 24, 4));
    lyrics.push_back(LyricSet("生きて踊ろう　僕らそれぞれの場所で", ofColor(text), 28, 5));
    lyrics.push_back(LyricSet("重なり合うよ", ofColor(text), 33, 2));
    
    lyrics.push_back(LyricSet("うちで歌おう　悲しみの向こう", ofColor(text), 35, 4));
    lyrics.push_back(LyricSet("全ての歌で　手を繋ごう", ofColor(text), 39, 4));
    lyrics.push_back(LyricSet("生きてまた会おう　僕らそれぞれの場所で", ofColor(text), 43, 5));
    lyrics.push_back(LyricSet("重なり合えそうだ", ofColor(text), 48, 5));
    
    start = false;
}


void Lyrics::update(ofSoundPlayer &sound)
{
    if (sound.isPlaying() && !start) {
        start = true;
        startTime = std::chrono::high_resolution_clock::now();
    }
}


void Lyrics::display()
{
    auto t1 = std::chrono::high_resolution_clock::now();
    int count = std::chrono::duration_cast<std::chrono::milliseconds>(t1 - startTime).count() / 1000.0;
    
    
    ofRectangle rect(0, ofGetHeight() / 2 + 190, ofGetWidth(), 80);
    
    ofPushStyle();
    ofFill();
    ofSetColor(base);
    ofDrawRectangle(rect);
    ofPopStyle();
    
    for (LyricSet &l : lyrics) {
        if (l.startCount <= count && count < (l.startCount + l.period)) {

            std::vector<ofPath> paths = font.getStringAsPoints(l.lyric, true, false);
            ofPushMatrix();
            ofSetColor(l.color);
            ofTranslate(ofGetWidth() / 2 - 400, ofGetHeight() / 2 + 250);
            for (ofPath p : paths) {
                p.draw();
            }
            ofPopMatrix();
            break;
        }
    }
}

Link to the reference page

ソースコードで使用したAPIの中から要点になりそうなものをいくつか選んでリストアップしました。

Development environment

• openframeworks 0.10.1
• c++
• macOS
• Xcode