Friday, July 11, 2008

Image Morphing with OpenCV

Avendo dovuto recentemente affrontare questo problema e non trovando nessun codice funzionante, ho pensato di pubblicare il codice da me prodotto prendendo spunto da quel poco che ho trovato.
Funziona così:
- si devono individuare sulle due immagini caricate 8 punti corrispondenti selezionandole con il mouse e partendo dall'immagine di sinistra.
- l'algoritmo calcola così la matrice Fondamentale.
- tramite tasto si fa variare il valore del parametro ALPHA. Con 0 si ottiene l'immagine di sinistra, con 1 quella di destra.
- si possono salvare e caricare i punti delle immagini selezionati

Se ci sono problemi sono qui.

/*
c --> save points
f --> calcultate morphing
l --> load points

*/

#include <cv.h>
#include <iostream>
#include <fstream>
#include <stdlib.h>
#include <vector>
#include "stdio.h"
#include "highgui.h"
#include <cvaux.h>
#include <cvtypes.h>

using namespace std;

int lrx, lry;
vector <CvPoint> allPoints;

CvMat* fundMat;
CvMat* points1;
CvMat* points2;

int counter = 16;
float alpha = 0;

void on_mousel( int event, int x, int y, int flags, void* param ){

if( event == CV_EVENT_LBUTTONDOWN){
cout << "Point " << counter / 2 << " left >> " << x << ' ' << y << endl;

if (counter % 2 == 0 && counter > 0){
CvPoint tmp = cvPoint(x,y);
allPoints.push_back(tmp);
counter--;
}else{
cout << "you have selected the wrong image" << endl;
}
}

}

int calcFundamental(){

if(allPoints.size() != 16) return -1;

// preparo la matrice delle corrispodenze

points1 = cvCreateMat(2,allPoints.size()/2,CV_32F);
points2 = cvCreateMat(2,allPoints.size()/2,CV_32F);
CvMat* status = cvCreateMat(1,allPoints.size()/2,CV_32F);

for(int j = 0; j < allPoints.size(); j++){

if(j%2==0){
cout << allPoints.at(j).x << ' ' << allPoints.at(j).y << endl;
cvSetReal2D(points1,0,j-(int)round(j/2),allPoints.at(j).x);
cvSetReal2D(points1,1,j-(int)round(j/2),allPoints.at(j).y);
}else{
cout << allPoints.at(j).x << ' ' << allPoints.at(j).y << endl;
cvSetReal2D(points2,0,j-(int)round(j/2)-1,allPoints.at(j).x);
cvSetReal2D(points2,1,j-(int)round(j/2)-1,allPoints.at(j).y);
}
}

fundMat = cvCreateMat(3,3,CV_32F);

int num = cvFindFundamentalMat(points1,points2,fundMat,CV_FM_8POINT,1.0,0.9999,status);

return num;
}

void loadPt(){
allPoints.clear();
FILE* inFile[2];
inFile[0] = fopen("leftPoint.txt","r");
inFile[1] = fopen("rightPoint.txt","r");
int num = 8;
CvPoint tmp;
while(num > 0){
for(int i = 0; i < 2; i++){

double x , y;

fscanf(inFile[i],"%lf %lf", &x, &y);
cout << x << ' ' << y << endl;
tmp.x = (int)x;
tmp.y = (int)y;
allPoints.push_back(tmp);
}
num --;
}
cout << "done " << endl;
}

void on_mouser( int event, int x, int y, int flags, void* param ){

if( event == CV_EVENT_LBUTTONDOWN){
cout << "Point " << round(counter / 2 + 0.5) << " right >> " << x << ' ' << y << endl;

if (counter % 2 == 1 && counter > 0){
CvPoint tmp = cvPoint(x,y);
allPoints.push_back(tmp);
counter--;
}else{
cout << "you have selected the wrong image" << endl;
}
}

}


int main(int argc, char** argv){

// per individuare le corrispondenze

IplImage* morphImage, *leftImage, *rightImage;

leftImage = cvLoadImage("FIRST_IMAGE.jpg");
rightImage = cvLoadImage("SECOND_IMAGE.jpg");

cvNamedWindow("left",1);
cvMoveWindow("left",10,10);
cvNamedWindow("right",1);
cvMoveWindow("right",400,10);

cvSetMouseCallback( "left", on_mousel, 0 );
cvSetMouseCallback( "right", on_mouser, 0 );

lrx = 0;
lry = 0;

while(1){

int i = 0;

while(i < allPoints.size()){
CvScalar s = cvScalar(0+rand()%255,0+rand()%255,0+rand()%255);
if(i%2==0){
CvPoint tmpl = cvPoint(allPoints.at(i).x,allPoints.at(i).y);
i++;
cvCircle(leftImage,tmpl,2,s,2);
}else{
CvPoint tmpr = cvPoint(allPoints.at(i).x,allPoints.at(i).y);
i++;
cvCircle(rightImage,tmpr,2,s,2);
}
}

cvShowImage("left",leftImage);
cvShowImage("right",rightImage);

int c = cvWaitKey(300);

if (c == 27) break;
else if (c == 99 && counter == 0){ // load saved points from files

//cout << allPoints.size()<< endl;
ofstream filel;
filel.open("leftPoint.txt",ios::out);

ofstream filer;
filer.open("rightPoint.txt",ios::out);

for(int j = 0; j < allPoints.size(); j++){
if(j%2==0){
filel << allPoints.at(j).x << " " << allPoints.at(j).y << "\n";
}else{
filer << allPoints.at(j).x << " " << allPoints.at(j).y << "\n";
}
}

filel.close();
filer.close();
allPoints.clear();
counter = 16;
cout << "points saved! " << endl;

}
else if(c == 102){ // calculare morphing
cout << calcFundamental()<< endl;

CvMat* epilines1 = cvCreateMat(3,8,CV_32F);
CvMat* epilines2 = cvCreateMat(3,8,CV_32F);

cvComputeCorrespondEpilines(points1,1,fundMat,epilines1); // calcola le epilines per ogni punto e resitituisce coeff a b c
cvComputeCorrespondEpilines(points2,2,fundMat,epilines2); // calcola le epilines per ogni punto e resitituisce coeff a b c

int lineCount; // numero di scanlines

static CvMatrix3 matrix;

float m00 = cvmGet(fundMat,0,0);
float m01 = cvmGet(fundMat,0,1);
float m02 = cvmGet(fundMat,0,2);
float m10 = cvmGet(fundMat,1,0);
float m11 = cvmGet(fundMat,1,1);
float m12 = cvmGet(fundMat,1,2);
float m20 = cvmGet(fundMat,2,0);
float m21 = cvmGet(fundMat,2,1);
float m22 = cvmGet(fundMat,2,2);


matrix.m[0][0] = m00;
matrix.m[0][1] = m01;
matrix.m[0][2] = m02;
matrix.m[1][0] = m10;
matrix.m[1][1] = m11;
matrix.m[1][2] = m12;
matrix.m[2][0] = m20;
matrix.m[2][1] = m21;
matrix.m[2][2] = m22;

CvMatrix3* matScan = &matrix;

cvMakeScanlines(matScan,cvSize(leftImage->width,leftImage->height),0,0,0,0,&lineCount); // calcolo il numero di scanlines

cout << lineCount << endl;

int* lengthEpilines1 = new int[lineCount]; // lunghezza delle rette epipolari
int* lengthEpilines2 = new int[lineCount];

int* epilinesInt1 = new int[4*lineCount]; // cordinate delle rette
int* epilinesInt2 = new int[4*lineCount];

cvMakeScanlines(matScan,cvSize(leftImage->width,leftImage->height),epilinesInt1,epilinesInt2,lengthEpilines1,lengthEpilines2,&lineCount);

//for(int j = 0; j <


uchar* preWarpData1 = new uchar[max(leftImage->width,leftImage->height)*lineCount*3]; // alloco spazio richiesto
uchar* preWarpData2 = new uchar[max(leftImage->width,leftImage->height)*lineCount*3];

cout << "warping " << endl;

cvPreWarpImage(lineCount,leftImage,preWarpData1,lengthEpilines1,epilinesInt1);
cvPreWarpImage(lineCount,rightImage,preWarpData2,lengthEpilines2,epilinesInt2);

cout << "done " << endl;

cvNamedWindow("right1",1);
cvShowImage("right1",rightImage);
cvNamedWindow("left1",1);
cvShowImage("left1",leftImage);

// cvShowImage("right",rightImage);

//cvWaitKey();

int* numRuns1 = new int[lineCount];
int* numRuns2 = new int[lineCount];

int* runs1 = new int[leftImage->width*lineCount];
int* runs2 = new int[leftImage->width*lineCount];

int* runCorrelation1 = new int[max(leftImage->width,leftImage->height)*lineCount*3];
int* runCorrelation2 = new int[max(leftImage->width,leftImage->height)*lineCount*3];

cvFindRuns(lineCount, preWarpData1, preWarpData2, lengthEpilines1, lengthEpilines2, runs1, runs2, numRuns1, numRuns2);

int* scanlinesMorphedImage = new int[lineCount*2*4];
int* numScanlinesMorphedImage = new int[lineCount*2*4];

cout << "runs " << endl;

cvDynamicCorrespondMulti(lineCount, runs1, numRuns1, runs2, numRuns2, runCorrelation1, runCorrelation2);

cout << "dyn " << endl;

uchar* tmpDataImageDst = new uchar[max(leftImage->width,leftImage->height)*lineCount*3];

//alpha = 0.0;

int* scanlinesA = new int[lineCount*2*4];
int* lenghts = new int[lineCount*2*4];

cvMakeAlphaScanlines(epilinesInt1, epilinesInt2, scanlinesMorphedImage, numScanlinesMorphedImage ,lineCount, alpha);

cvMorphEpilinesMulti(lineCount, preWarpData1, lengthEpilines1, preWarpData2, lengthEpilines2, tmpDataImageDst, numScanlinesMorphedImage, alpha, runs1, numRuns1, runs2, numRuns2, runCorrelation1, runCorrelation2);

cout << "morph " << endl;

morphImage = cvCreateImage(cvSize(leftImage->width,leftImage->height),8,3);

cvPostWarpImage(lineCount, tmpDataImageDst, numScanlinesMorphedImage, morphImage, scanlinesMorphedImage);

cvDeleteMoire(morphImage);

cvNamedWindow("altro",1);
cvShowImage("altro",morphImage);

cout << "alpha value: " << alpha << endl;
alpha += 0.1;


}else if(c == 108){
cout << "load points " << endl;
loadPt();
}
}
}





2 comments:

Cliff said...

cvComputeCorrespondEpilines does not seem to be used in your code. Is there a piece missing or is this supposed to be unused?

iPhoneCoder said...

Hi. Thanks for the awesome tutorial. I am following all the steps as mentioned here but getting noise in the in-between images. Please let me know how to remove the noise. I can also share a sample image if required. Thanks again :)