To categorize big, middle, small of camera movement using k-means

Mr. Juan ask me that to categorize camera movement rate into big, middle, small.
He is researching to find meaningful scene in endoscopic video.

His one of approach is using optical flow.
refer to this page.
http://study.marearts.com/2015/01/the-method-to-check-that-camera-is.html
Dense optical flow can know how much video moving.
The source code of reference page checked camera move or not by the percentage of movement.
There are 2 threshold values.
Count number of moved pixels, this is counted when pixels moved over than threshold-1 value.
And check whether the counted pixels is over than threshold-2 percent or not in all pixels.

By the way, he ask me to separate video moving by big, middle, small using k-means clustering algorithm without threshold.
In past, I introduced the usage of k-mean algorithm using openCV.
See this page.
http://study.marearts.com/search/label/K-means


This work is separated by 3 steps.
Step 1 is to calculate movement rate in video using optical flow.
And save fame and movement rate information to txt file.

Step 2 is clustering by 3 class degree movement using k-means.
read step 1 file and write clustering information to txt file.

Step 3 is for display.
read step 3 txt file and display moving rate on the video.

refer to these source code.
Thank you.

Step 1.
Getting movement rate of frames.


moving rate is calculated.

...
#include < stdio.h>  

#include < opencv2\opencv.hpp>  
#include < opencv2/core/core.hpp>  
#include < opencv2/highgui/highgui.hpp>  
#include < opencv2\gpu\gpu.hpp>  
#include < opencv2\nonfree\features2d.hpp >      



#ifdef _DEBUG          
#pragma comment(lib, "opencv_core249d.lib")  
#pragma comment(lib, "opencv_imgproc249d.lib")   //MAT processing  
#pragma comment(lib, "opencv_objdetect249d.lib") //HOGDescriptor  
#pragma comment(lib, "opencv_gpu249d.lib")  
#pragma comment(lib, "opencv_features2d249d.lib")  
#pragma comment(lib, "opencv_highgui249d.lib")  
#else  
#pragma comment(lib, "opencv_core249.lib")  
#pragma comment(lib, "opencv_imgproc249.lib")  
#pragma comment(lib, "opencv_objdetect249.lib")  
#pragma comment(lib, "opencv_gpu249.lib")  
#pragma comment(lib, "opencv_features2d249.lib")  
#pragma comment(lib, "opencv_highgui249.lib")  
#endif   

using namespace std;  
using namespace cv;  

#define WIDTH_DENSE (80)  
#define HEIGHT_DENSE (60)  

#define DENSE_DRAW 0 //dense optical flow arrow drawing or not  
#define GLOBAL_MOTION_TH1 1  
#define GLOBAL_MOTION_TH2 70  


float drawOptFlowMap_gpu (const Mat& flow_x, const Mat& flow_y, Mat& cflowmap, int step, float scaleX, float scaleY, int drawOnOff);  


int main()  
{  
 //stream /////////////////////////////////////////////////  
 VideoCapture stream1("M:\\____videoSample____\\medical\\HUV-03-14.wmv");

 //variables /////////////////////////////////////////////  
 Mat O_Img; //Mat  
 gpu::GpuMat O_Img_gpu; //GPU  
 gpu::GpuMat R_Img_gpu_dense; //gpu dense resize  
 gpu::GpuMat R_Img_gpu_dense_gray_pre; //gpu dense resize gray  
 gpu::GpuMat R_Img_gpu_dense_gray; //gpu dense resize gray  
 gpu::GpuMat flow_x_gpu, flow_y_gpu;  
 Mat flow_x, flow_y;  

 //algorithm *************************************  
 //dense optical flow  
 gpu::FarnebackOpticalFlow fbOF;  


 //running once //////////////////////////////////////////  
 if(!(stream1.read(O_Img))) //get one frame form video  
 {  
  printf("Open Fail !!\n");  
  return 0;   
 }  

 //for rate calucation  
 float scaleX, scaleY;  
 scaleX = O_Img.cols/WIDTH_DENSE;  
 scaleY = O_Img.rows/HEIGHT_DENSE;  

 O_Img_gpu.upload(O_Img);   
 gpu::resize(O_Img_gpu, R_Img_gpu_dense, Size(WIDTH_DENSE, HEIGHT_DENSE));  
 gpu::cvtColor(R_Img_gpu_dense, R_Img_gpu_dense_gray_pre, CV_BGR2GRAY);  


 //////////////////////////////////////////////////////////
 FILE *fp = fopen("DataOutput.txt","w");

 //unconditional loop   ///////////////////////////////////  
 int frame=0;
 int untilFrame=1000;
 while (true) {  
  frame++;
  if(frame>untilFrame)  //stop point.
   break;

  //reading  
  if( stream1.read(O_Img) == 0) //get one frame form video     
   break;  

  // ---------------------------------------------------  
  //upload cou mat to gpu mat  
  O_Img_gpu.upload(O_Img);   
  //resize  
  gpu::resize(O_Img_gpu, R_Img_gpu_dense, Size(WIDTH_DENSE, HEIGHT_DENSE));  
  //color to gray  
  gpu::cvtColor(R_Img_gpu_dense, R_Img_gpu_dense_gray, CV_BGR2GRAY);  

  //calculate dense optical flow using GPU version  
  fbOF.operator()(R_Img_gpu_dense_gray_pre, R_Img_gpu_dense_gray, flow_x_gpu, flow_y_gpu);  
  flow_x_gpu.download( flow_x );  
  flow_y_gpu.download( flow_y );  


  //calculate motion rate in whole image  
  float motionRate = drawOptFlowMap_gpu(flow_x, flow_y, O_Img, 1, scaleX, scaleY, DENSE_DRAW);  
  //update pre image  
  R_Img_gpu_dense_gray_pre = R_Img_gpu_dense_gray.clone();  



  //display "moving rate (0~100%)" and save to txt with frame
  char TestStr[100];  
  sprintf(TestStr, "%.2lf %% moving", motionRate);
  putText(O_Img, TestStr, Point(30,60), CV_FONT_NORMAL, 1, Scalar(255,255,255),2,2); //OutImg is Mat class;     
  
  //output "frame, motionRate" to txt
  fprintf(fp,"%d %.2lf\n", frame, motionRate);

  // show image ----------------------------------------  
  imshow("Origin", O_Img);     

  // wait key  
  if( cv::waitKey(100) > 30)  
   break;  
 }  

 fclose(fp);
}  



float drawOptFlowMap_gpu (const Mat& flow_x, const Mat& flow_y, Mat& cflowmap, int step, float scaleX, float scaleY, int drawOnOff)  
{  
 double count=0;  

 float countOverTh1 = 0;  
 int sx,sy;  
 for(int y = 0; y < HEIGHT_DENSE; y += step)  
 {  
  for(int x = 0; x < WIDTH_DENSE; x += step)  
  {  

   if(drawOnOff)  
   {  
    Point2f fxy;      
    fxy.x = cvRound( flow_x.at< float >(y, x)*scaleX + x*scaleX );     
    fxy.y = cvRound( flow_y.at< float >(y, x)*scaleY + y*scaleY );     
    line(cflowmap, Point(x*scaleX,y*scaleY), Point(fxy.x, fxy.y), CV_RGB(0, 255, 0));     
    circle(cflowmap, Point(fxy.x, fxy.y), 1, CV_RGB(0, 255, 0), -1);     
   }  

   float xx = fabs(flow_x.at< float >(y, x) );  
   float yy = fabs(flow_y.at< float >(y, x) );  

   float xxyy = sqrt(xx*xx + yy*yy);  
   if( xxyy > GLOBAL_MOTION_TH1 )  
    countOverTh1 = countOverTh1 +1;  

   count=count+1;  
  }  
 }  
 return (countOverTh1 / count) * 100;  

}  

...


step 2. clustering movement rate to 3 classes.
The movement rates are clustered by 3 values

Frames and class ID

...
#include <  stdio.h>     
#include <  iostream>     
#include <  opencv2\opencv.hpp>     


#ifdef _DEBUG             
#pragma comment(lib, "opencv_core249d.lib")     
#pragma comment(lib, "opencv_imgproc249d.lib")   //MAT processing     
#pragma comment(lib, "opencv_highgui249d.lib")     
#else     
#pragma comment(lib, "opencv_core249.lib")     
#pragma comment(lib, "opencv_imgproc249.lib")     
#pragma comment(lib, "opencv_highgui249.lib")     
#endif    

using namespace cv;  
using namespace std;  

void main()  
{  

 //read data
 FILE* fp = fopen("DataOutput.txt","r");
 vector< float > readDataV;
 int frames;
 double movingRate;
 while(fscanf(fp,"%d %lf", &frames, &movingRate) != EOF )
 {
  readDataV.push_back( movingRate );
  //printf("%d %lf \n", frames, movingRate);
 }
 fclose(fp);
 
 //preparing variables for kmeans
 Mat samples(readDataV.size(), 1, CV_32F);  
 //copy vector to mat
 memcpy(samples.data, readDataV.data(), readDataV.size()*sizeof(float) );

 //kmean
 int clusterCount = 3;
 Mat labels;
 int attempts = 10;
 Mat centers;
 kmeans(samples, clusterCount, labels, 
  TermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 10, 1.0), 
  attempts, KMEANS_RANDOM_CENTERS, centers );

 //result out
 //frames, class index
 for(int i=0; i< clusterCount; ++i)
 {
  printf("%d class center %lf\n",  i, centers.at< float>(i,0) );
 }
 printf("\n");
 
 
 FILE* fp2 = fopen("ResultKmeans.txt", "w");
 for(int i=0; i< readDataV.size(); ++i)
 {
  //printf("%d %d\n", i, labels.at< int>(i,0) );
  fprintf(fp2, "%d %d\n", i+1, labels.at< int>(i,0) );
 }
 fclose(fp2);

}  
...

step 3. display movement labels on the video.

display 3 type of movement category(big, middle, small).

...
#include < time.h>
#include < opencv2\opencv.hpp>
#include < string>
#include < stdio.h>


#ifdef _DEBUG        
#pragma comment(lib, "opencv_core249d.lib")
#pragma comment(lib, "opencv_imgproc249d.lib")   //MAT processing
#pragma comment(lib, "opencv_highgui249d.lib")
#else
#pragma comment(lib, "opencv_core249.lib")
#pragma comment(lib, "opencv_imgproc249.lib")
#pragma comment(lib, "opencv_highgui249.lib")
#endif   

using namespace std;
using namespace cv;

int main()
{
 /////////////////////////////////////////////////////////////////////////
 //read file
 FILE* fp = fopen("ResultKmeans.txt","r");
 vector< int > readDataV;
 int frames;
 int labels;
 while(fscanf(fp,"%d %d", &frames, &labels) != EOF )
 {
  readDataV.push_back( labels );
  //printf("%d %d \n", frames, labels);
 }
 fclose(fp);


 //Load avi file 
 VideoCapture stream1("M:\\____videoSample____\\medical\\HUV-03-14.wmv");
 /////////////////////////////////////////////////////////////////////////

 //Mat and GpuMat
 Mat o_frame; 

 //capture
 stream1 >> o_frame;
 if( o_frame.empty() )
   return 0; 
 //////////////////////////////////////////////////////////////////////////

 int frame=0;
 int untilFrame=1000;
 while(1)
 {
  frame++;
  if(frame>untilFrame)  //stop point.
   break;

  /////////////////////////////////////////////////////////////////////////
  stream1 >> o_frame;
  if( o_frame.empty() )
   return 0;


  char TestStr[100];    
  if(readDataV[frame-1] == 0)  
   sprintf(TestStr, "Big moving");   
  else if(readDataV[frame-1] == 1)
   sprintf(TestStr, "middle moving");
  else
   sprintf(TestStr, "small moving");

  putText(o_frame, TestStr, Point(30,60), CV_FONT_NORMAL, 1, Scalar(255,255,255),2,2); //OutImg is Mat class;     


  //Display   
  imshow("origin", o_frame);  
  /////////////////////////////////////////////////////////////////////////

  if( waitKey(10) > 0)
   break;
 }

 return 0;
}


...











Comments

Popular posts from this blog

OpenCV Stitching example (Stitcher class, Panorama)

(OpenCV Study) Background subtractor MOG, MOG2, GMG example source code (BackgroundSubtractorMOG, BackgroundSubtractorMOG2, BackgroundSubtractorGMG)

Example source code of extract HOG feature from images, save descriptor values to xml file, using opencv (using HOGDescriptor )

Real-time N camera stitching Class.

8 point algorithm (Matlab source code) / The method to get the Fundamental Matrix and the Essential matrix

Optical Flow sample source code using OpenCV

Video Stabilization example source code, (using cvFindHomography, cvWarpPerspective functions in openCV)

(OpenCV Study) calcOpticalFlowFarneback example source code ( dense optical flow )

yuv422(YUYV) to RGB and RGB to yuv422(YUYV), (Using OpenCV and TBB)

OpenCV Drawing Example, (line, circle, rectangle, ellipse, polyline, fillConvexPoly, putText, drawContours)