9/22/2015

opencv rtsp connection using vlc library

RTSP(Real Time Streaming Protocol) is video streaming, it usually sent from network camera.

VideoCapture function in opencv also can get rtsp video, but difficult to receive a stable image.
This is simple example using videocapture.
...
int main()
{

 VideoCapture cap("rtsp://192.168.0.50");

 namedWindow("fish", 0);
 Mat fishEye;
 while (1)
 {
  cap >> fishEye;
  if (fishEye.empty())
   return 0;

  imshow("fish", fishEye);
  if (waitKey(10) > 0)
   break;
 }

 return 0;

}
...

We can receive more reliable RTSP using VLC library.
Of course, there are many different ways(LIVE555, FFmpeg, Qt api...).

#include < windows.h>
#include < vlc/vlc.h>

#include < opencv2\opencv.hpp>    
#include < opencv2\core.hpp>  
#include < opencv2\highgui.hpp>   

#pragma comment(lib, "libvlc.lib")

#ifdef _DEBUG
#pragma comment(lib, "opencv_core300d.lib")  
#pragma comment(lib, "opencv_highgui300d.lib")
#pragma comment(lib, "opencv_imgproc300d.lib")
#pragma comment(lib, "opencv_imgcodecs300d.lib")
#else
#pragma comment(lib, "opencv_core300.lib")  
#pragma comment(lib, "opencv_highgui300.lib")
#pragma comment(lib, "opencv_imgproc300.lib")
#pragma comment(lib, "opencv_imgcodecs300.lib")
#endif

using namespace cv;
using namespace std;

struct ctx
{
 Mat* image;
 HANDLE mutex;
 uchar* pixels;
};
bool isRunning = true;

Size getsize(const char* path)
{
 libvlc_instance_t *vlcInstance;
 libvlc_media_player_t *mp;
 libvlc_media_t *media;

 char *vlc_argv[6];
 vlc_argv[0] = new char[3];
 strcpy_s(vlc_argv[0], 3, "-I");
 vlc_argv[1] = new char[6];
 strcpy_s(vlc_argv[1], 6, "dummy"); // Don't use any interface
 vlc_argv[2] = new char[16];
 strcpy_s(vlc_argv[2], 16, "--ignore-config"); // Don't use VLC's config
 vlc_argv[3] = new char[128];
 strcpy_s(vlc_argv[3], 128, "--plugin-path=/plugins");
 int vlc_argc = 4;

 vlcInstance = libvlc_new(vlc_argc, vlc_argv);

 for (int i = 0; i < vlc_argc; i++)
  delete[] vlc_argv[i];

 media = libvlc_media_new_location(vlcInstance, path);
 mp = libvlc_media_player_new_from_media(media);

 libvlc_media_release(media);
 libvlc_video_set_callbacks(mp, NULL, NULL, NULL, NULL);
 libvlc_video_set_format(mp, "RV24", 100, 100, 100 * 24 / 8); // pitch = width * BitsPerPixel / 8
 //libvlc_video_set_format(mp, "RV32", 100, 100, 100 * 4);
 libvlc_media_player_play(mp);

 Sleep(2000);//wait a while so that something get rendered so that size info is available
 unsigned int width = 640, height = 480;

 for (int i = 0; i < 30 && height == 0; i++)
 {
  Sleep(50);
  libvlc_video_get_size(mp, 0, &width, &height);

  if (width != 0 && height != 0)
   break;
 }


 if (width == 0 || height == 0)
 {
  width = 640;
  height = 480;
 }

 libvlc_media_player_stop(mp);
 libvlc_release(vlcInstance);
 libvlc_media_player_release(mp);
 return Size(width, height);
}


void *lock(void *data, void**p_pixels)
{
 struct ctx *ctx = (struct ctx*)data;
 WaitForSingleObject(ctx->mutex, INFINITE);
 *p_pixels = ctx->pixels;
 return NULL;

}

void display(void *data, void *id){
 (void)data;
 assert(id == NULL);
}

void unlock(void *data, void *id, void *const *p_pixels)
{

 struct ctx *ctx = (struct ctx*)data;
 Mat frame = *ctx->image;
 if (frame.data)
 {
  imshow("frame", frame);
  if (waitKey(1) == 27)
  {
   isRunning = false;
   //exit(0);
  }
 }
 ReleaseMutex(ctx->mutex);
}


int main()
{
 string url = "rtsp://...";
 //vlc sdk does not know the video size until it is rendered, so need to play it a bit so that size is     known
 Size sz = getsize(url.c_str());

 // VLC pointers
 libvlc_instance_t *vlcInstance;
 libvlc_media_player_t *mp;
 libvlc_media_t *media;

 char *vlc_argv[6];
 vlc_argv[0] = new char[3];
 strcpy_s(vlc_argv[0], 3, "-I");
 vlc_argv[1] = new char[6];
 strcpy_s(vlc_argv[1], 6, "dummy"); // Don't use any interface
 vlc_argv[2] = new char[16];
 strcpy_s(vlc_argv[2], 16, "--ignore-config"); // Don't use VLC's config
 vlc_argv[3] = new char[128];
 strcpy_s(vlc_argv[3], 128, "--plugin-path=/plugins");
 int vlc_argc = 4;

 vlcInstance = libvlc_new(vlc_argc, vlc_argv);

 for (int i = 0; i < vlc_argc; i++)
  delete[] vlc_argv[i];

 media = libvlc_media_new_location(vlcInstance, url.c_str());
 mp = libvlc_media_player_new_from_media(media);
 libvlc_media_release(media);


 struct ctx* context = (struct ctx*)malloc(sizeof(*context));
 context->mutex = CreateMutex(NULL, FALSE, NULL);
 context->image = new Mat(sz.height, sz.width, CV_8UC3);
 context->pixels = (unsigned char *)context->image->data;

 libvlc_video_set_callbacks(mp, lock, unlock, display, context);
 libvlc_video_set_format(mp, "RV24", sz.width, sz.height, sz.width * 24 / 8); // pitch = width *     BitsPerPixel / 8
 //libvlc_video_set_format(mp, "RV32", sz.width, sz.height, sz.width * 4);

 libvlc_media_player_play(mp);
 while (isRunning)
 {
  //imshow("rtsp", *(context->image));
  Sleep(1);  
 }

 libvlc_media_player_stop(mp);
 libvlc_release(vlcInstance);
 libvlc_media_player_release(mp);
 free(context);

 return 0;
}


This example code is referenced in here
http://stackoverflow.com/questions/23529620/opencv-and-network-cameras-or-how-to-spy-on-the-neighbors

To use this example code, you should install VLC player, and set dependency referencing include, lib, dll and plugins.
And If you are using opencv 64 bits then you also install 64-bit vlc.

Test video


9/09/2015

opencv 3.0 cascade_gpu example source code

opencv 2.49 example is here
http://study.marearts.com/2014/09/opencv-face-detection-using-adaboost.html



#include < iostream>    
#include "opencv2\objdetect\objdetect.hpp"
#include "opencv2\highgui\highgui.hpp"
#include "opencv2\imgproc\imgproc.hpp"
#include "opencv2\cudaobjdetect.hpp"
#include "opencv2\cudaimgproc.hpp"

#ifdef _DEBUG               
#pragma comment(lib, "opencv_core300d.lib")       
#pragma comment(lib, "opencv_highgui300d.lib")    
#pragma comment(lib, "opencv_imgcodecs300d.lib")  
#pragma comment(lib, "opencv_objdetect300d.lib")  
#pragma comment(lib, "opencv_imgproc300d.lib")  
#pragma comment(lib, "opencv_cudaobjdetect300d.lib")  
#else       
#pragma comment(lib, "opencv_core300.lib")       
#pragma comment(lib, "opencv_highgui300.lib")    
#pragma comment(lib, "opencv_imgcodecs300.lib")    
#pragma comment(lib, "opencv_objdetect300.lib")  
#pragma comment(lib, "opencv_imgproc300.lib")  
#pragma comment(lib, "opencv_cudaobjdetect300.lib")  
#endif        

using namespace std;
using namespace cv;


void main()
{
 //for time measure  
 float TakeTime;
 unsigned long Atime, Btime;

 //window  
 namedWindow("origin");

 //load image  
 Mat img = imread("sh.jpg");
 Mat grayImg; //adaboost detection is gray input only.  
 cvtColor(img, grayImg, CV_BGR2GRAY);

 //load xml file  
 string trainface = ".\\haarcascade_frontalface_alt.xml";

 //declaration  
 Ptr< cuda::CascadeClassifier> cascade_gpu = cuda::CascadeClassifier::create(trainface);

 
 /////////////////////////////////////////////  
 
 //gpu case face detection code  
 cuda::GpuMat faceBuf_gpu;
 cuda::GpuMat GpuImg;
 vector< Rect> faces;

 GpuImg.upload(grayImg);
 Atime = getTickCount();

 cascade_gpu->detectMultiScale(GpuImg, faceBuf_gpu);
 cascade_gpu->convert(faceBuf_gpu, faces);
 Btime = getTickCount();
 TakeTime = (Btime - Atime) / getTickFrequency();
 printf("detected face(gpu version) =%d / %lf sec take.\n", faces.size(), TakeTime);
 Mat faces_downloaded;
 if (faces.size() >= 1)
 {
  for (size_t i = 0; i < faces.size(); ++i)
   rectangle(img, faces[i], Scalar(255));
 } 

 /////////////////////////////////////////////////  
 //result display  
 imshow("origin", img);
 waitKey(0);
}


mnist image dataset (jpg files)

The MNIST dataset is a dataset of handwritten digits, comprising 60 000 training examples and 10 000 test examples. The dataset can be downloaded from http://yann.lecun.com/exdb/mnist/.
MNIST data is provided in binary file for user convenience.

But, in my case, this binary type is more difficult.
I am needed image files.

I want to start whole process of deep learning(make leveldb file, learning, classify) using MNIST data and compare to example result in caffe model.


This site tells me a hint how to make image datas.

This is my matlab code for making jpg file.

---
%%
clc;
clear all;

% Change the filenames if you've saved the files under different names
% On some platforms, the files might be saved as
% train-images.idx3-ubyte / train-labels.idx1-ubyte


images = loadMNISTImages('t10k-images.idx3-ubyte');
labels = loadMNISTLabels('t10k-labels.idx1-ubyte');

%images = loadMNISTImages('train-images.idx3-ubyte');
%labels = loadMNISTLabels('train-labels.idx1-ubyte');

% We are using display_network from the autoencoder code
display_network(images(:,1:60000)); % Show the first 100 images
disp(labels(1:1));

% save one digit number to jpg file
%{
[w h]=size(images)
endIndex = h;

dindex=zeros(10,1);
for j=1:endIndex
aImg = reshape( images(:,j), 28, 28);
imshow(aImg);
label = labels(j);
dindex( label+1 ) = dindex( label+1 ) +1;
dataD = strcat('./data2/mnist_', num2str( label ), '_', num2str( dindex(label+1) ), '.jpg' );
%dindex
imwrite(aImg, dataD);
dataD
end

fileID = fopen('test.txt','w');
%make path and label
dindex=zeros(10,1);
for j=1:endIndex
label = labels(j);
dindex( label+1 ) = dindex( label+1 ) +1;
dataD = strcat('/data2/mnist_', num2str( label ), '_', num2str( dindex(label+1) ), '.jpg' );
fprintf(fileID,'%s %d\n',dataD, label);
end
fclose(fileID);
%}
---

And images of mnist files


If you need images of mnist, request to me, I will send by email.
The file size is approximately 37 MB.

Thank you.

You can download all jpeg images and origin binary data from here: