Mr. Juan ask me that to categorize camera movement rate into big, middle, small.
He is researching to find meaningful scene in endoscopic video.
His one of approach is using optical flow.
refer to this page.
http://study.marearts.com/2015/01/the-method-to-check-that-camera-is.html
Dense optical flow can know how much video moving.
The source code of reference page checked camera move or not by the percentage of movement.
There are 2 threshold values.
Count number of moved pixels, this is counted when pixels moved over than threshold-1 value.
And check whether the counted pixels is over than threshold-2 percent or not in all pixels.
By the way, he ask me to separate video moving by big, middle, small using k-means clustering algorithm without threshold.
In past, I introduced the usage of k-mean algorithm using openCV.
See this page.
http://study.marearts.com/search/label/K-means
This work is separated by 3 steps.
Step 1 is to calculate movement rate in video using optical flow.
And save fame and movement rate information to txt file.
Step 2 is clustering by 3 class degree movement using k-means.
read step 1 file and write clustering information to txt file.
Step 3 is for display.
read step 3 txt file and display moving rate on the video.
refer to these source code.
Thank you.
Step 1.
Getting movement rate of frames.
moving rate is calculated.
...
#include < stdio.h>
#include < opencv2\opencv.hpp>
#include < opencv2/core/core.hpp>
#include < opencv2/highgui/highgui.hpp>
#include < opencv2\gpu\gpu.hpp>
#include < opencv2\nonfree\features2d.hpp >
#ifdef _DEBUG
#pragma comment(lib, "opencv_core249d.lib")
#pragma comment(lib, "opencv_imgproc249d.lib") //MAT processing
#pragma comment(lib, "opencv_objdetect249d.lib") //HOGDescriptor
#pragma comment(lib, "opencv_gpu249d.lib")
#pragma comment(lib, "opencv_features2d249d.lib")
#pragma comment(lib, "opencv_highgui249d.lib")
#else
#pragma comment(lib, "opencv_core249.lib")
#pragma comment(lib, "opencv_imgproc249.lib")
#pragma comment(lib, "opencv_objdetect249.lib")
#pragma comment(lib, "opencv_gpu249.lib")
#pragma comment(lib, "opencv_features2d249.lib")
#pragma comment(lib, "opencv_highgui249.lib")
#endif
using namespace std;
using namespace cv;
#define WIDTH_DENSE (80)
#define HEIGHT_DENSE (60)
#define DENSE_DRAW 0 //dense optical flow arrow drawing or not
#define GLOBAL_MOTION_TH1 1
#define GLOBAL_MOTION_TH2 70
float drawOptFlowMap_gpu (const Mat& flow_x, const Mat& flow_y, Mat& cflowmap, int step, float scaleX, float scaleY, int drawOnOff);
int main()
{
//stream /////////////////////////////////////////////////
VideoCapture stream1("M:\\____videoSample____\\medical\\HUV-03-14.wmv");
//variables /////////////////////////////////////////////
Mat O_Img; //Mat
gpu::GpuMat O_Img_gpu; //GPU
gpu::GpuMat R_Img_gpu_dense; //gpu dense resize
gpu::GpuMat R_Img_gpu_dense_gray_pre; //gpu dense resize gray
gpu::GpuMat R_Img_gpu_dense_gray; //gpu dense resize gray
gpu::GpuMat flow_x_gpu, flow_y_gpu;
Mat flow_x, flow_y;
//algorithm *************************************
//dense optical flow
gpu::FarnebackOpticalFlow fbOF;
//running once //////////////////////////////////////////
if(!(stream1.read(O_Img))) //get one frame form video
{
printf("Open Fail !!\n");
return 0;
}
//for rate calucation
float scaleX, scaleY;
scaleX = O_Img.cols/WIDTH_DENSE;
scaleY = O_Img.rows/HEIGHT_DENSE;
O_Img_gpu.upload(O_Img);
gpu::resize(O_Img_gpu, R_Img_gpu_dense, Size(WIDTH_DENSE, HEIGHT_DENSE));
gpu::cvtColor(R_Img_gpu_dense, R_Img_gpu_dense_gray_pre, CV_BGR2GRAY);
//////////////////////////////////////////////////////////
FILE *fp = fopen("DataOutput.txt","w");
//unconditional loop ///////////////////////////////////
int frame=0;
int untilFrame=1000;
while (true) {
frame++;
if(frame>untilFrame) //stop point.
break;
//reading
if( stream1.read(O_Img) == 0) //get one frame form video
break;
// ---------------------------------------------------
//upload cou mat to gpu mat
O_Img_gpu.upload(O_Img);
//resize
gpu::resize(O_Img_gpu, R_Img_gpu_dense, Size(WIDTH_DENSE, HEIGHT_DENSE));
//color to gray
gpu::cvtColor(R_Img_gpu_dense, R_Img_gpu_dense_gray, CV_BGR2GRAY);
//calculate dense optical flow using GPU version
fbOF.operator()(R_Img_gpu_dense_gray_pre, R_Img_gpu_dense_gray, flow_x_gpu, flow_y_gpu);
flow_x_gpu.download( flow_x );
flow_y_gpu.download( flow_y );
//calculate motion rate in whole image
float motionRate = drawOptFlowMap_gpu(flow_x, flow_y, O_Img, 1, scaleX, scaleY, DENSE_DRAW);
//update pre image
R_Img_gpu_dense_gray_pre = R_Img_gpu_dense_gray.clone();
//display "moving rate (0~100%)" and save to txt with frame
char TestStr[100];
sprintf(TestStr, "%.2lf %% moving", motionRate);
putText(O_Img, TestStr, Point(30,60), CV_FONT_NORMAL, 1, Scalar(255,255,255),2,2); //OutImg is Mat class;
//output "frame, motionRate" to txt
fprintf(fp,"%d %.2lf\n", frame, motionRate);
// show image ----------------------------------------
imshow("Origin", O_Img);
// wait key
if( cv::waitKey(100) > 30)
break;
}
fclose(fp);
}
float drawOptFlowMap_gpu (const Mat& flow_x, const Mat& flow_y, Mat& cflowmap, int step, float scaleX, float scaleY, int drawOnOff)
{
double count=0;
float countOverTh1 = 0;
int sx,sy;
for(int y = 0; y < HEIGHT_DENSE; y += step)
{
for(int x = 0; x < WIDTH_DENSE; x += step)
{
if(drawOnOff)
{
Point2f fxy;
fxy.x = cvRound( flow_x.at< float >(y, x)*scaleX + x*scaleX );
fxy.y = cvRound( flow_y.at< float >(y, x)*scaleY + y*scaleY );
line(cflowmap, Point(x*scaleX,y*scaleY), Point(fxy.x, fxy.y), CV_RGB(0, 255, 0));
circle(cflowmap, Point(fxy.x, fxy.y), 1, CV_RGB(0, 255, 0), -1);
}
float xx = fabs(flow_x.at< float >(y, x) );
float yy = fabs(flow_y.at< float >(y, x) );
float xxyy = sqrt(xx*xx + yy*yy);
if( xxyy > GLOBAL_MOTION_TH1 )
countOverTh1 = countOverTh1 +1;
count=count+1;
}
}
return (countOverTh1 / count) * 100;
}
...
step 2. clustering movement rate to 3 classes.
The movement rates are clustered by 3 values
Frames and class ID
...
#include < stdio.h>
#include < iostream>
#include < opencv2\opencv.hpp>
#ifdef _DEBUG
#pragma comment(lib, "opencv_core249d.lib")
#pragma comment(lib, "opencv_imgproc249d.lib") //MAT processing
#pragma comment(lib, "opencv_highgui249d.lib")
#else
#pragma comment(lib, "opencv_core249.lib")
#pragma comment(lib, "opencv_imgproc249.lib")
#pragma comment(lib, "opencv_highgui249.lib")
#endif
using namespace cv;
using namespace std;
void main()
{
//read data
FILE* fp = fopen("DataOutput.txt","r");
vector< float > readDataV;
int frames;
double movingRate;
while(fscanf(fp,"%d %lf", &frames, &movingRate) != EOF )
{
readDataV.push_back( movingRate );
//printf("%d %lf \n", frames, movingRate);
}
fclose(fp);
//preparing variables for kmeans
Mat samples(readDataV.size(), 1, CV_32F);
//copy vector to mat
memcpy(samples.data, readDataV.data(), readDataV.size()*sizeof(float) );
//kmean
int clusterCount = 3;
Mat labels;
int attempts = 10;
Mat centers;
kmeans(samples, clusterCount, labels,
TermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 10, 1.0),
attempts, KMEANS_RANDOM_CENTERS, centers );
//result out
//frames, class index
for(int i=0; i< clusterCount; ++i)
{
printf("%d class center %lf\n", i, centers.at< float>(i,0) );
}
printf("\n");
FILE* fp2 = fopen("ResultKmeans.txt", "w");
for(int i=0; i< readDataV.size(); ++i)
{
//printf("%d %d\n", i, labels.at< int>(i,0) );
fprintf(fp2, "%d %d\n", i+1, labels.at< int>(i,0) );
}
fclose(fp2);
}
...
step 3. display movement labels on the video.
display 3 type of movement category(big, middle, small).
...
#include < time.h>
#include < opencv2\opencv.hpp>
#include < string>
#include < stdio.h>
#ifdef _DEBUG
#pragma comment(lib, "opencv_core249d.lib")
#pragma comment(lib, "opencv_imgproc249d.lib") //MAT processing
#pragma comment(lib, "opencv_highgui249d.lib")
#else
#pragma comment(lib, "opencv_core249.lib")
#pragma comment(lib, "opencv_imgproc249.lib")
#pragma comment(lib, "opencv_highgui249.lib")
#endif
using namespace std;
using namespace cv;
int main()
{
/////////////////////////////////////////////////////////////////////////
//read file
FILE* fp = fopen("ResultKmeans.txt","r");
vector< int > readDataV;
int frames;
int labels;
while(fscanf(fp,"%d %d", &frames, &labels) != EOF )
{
readDataV.push_back( labels );
//printf("%d %d \n", frames, labels);
}
fclose(fp);
//Load avi file
VideoCapture stream1("M:\\____videoSample____\\medical\\HUV-03-14.wmv");
/////////////////////////////////////////////////////////////////////////
//Mat and GpuMat
Mat o_frame;
//capture
stream1 >> o_frame;
if( o_frame.empty() )
return 0;
//////////////////////////////////////////////////////////////////////////
int frame=0;
int untilFrame=1000;
while(1)
{
frame++;
if(frame>untilFrame) //stop point.
break;
/////////////////////////////////////////////////////////////////////////
stream1 >> o_frame;
if( o_frame.empty() )
return 0;
char TestStr[100];
if(readDataV[frame-1] == 0)
sprintf(TestStr, "Big moving");
else if(readDataV[frame-1] == 1)
sprintf(TestStr, "middle moving");
else
sprintf(TestStr, "small moving");
putText(o_frame, TestStr, Point(30,60), CV_FONT_NORMAL, 1, Scalar(255,255,255),2,2); //OutImg is Mat class;
//Display
imshow("origin", o_frame);
/////////////////////////////////////////////////////////////////////////
if( waitKey(10) > 0)
break;
}
return 0;
}
...
VIDEO
