OpenCV RTSP receiving test

To get the RTSP video and showing is easy when we use VideoCapture function in OpenCV.
http://study.marearts.com/2015/05/opencv300-rc1-videocapture-example.html
http://study.marearts.com/2013/09/opencv-video-writer-example-source-code.html

But the video looks even broken video stream is shifted a bit.
So I use VLC library for stable receiving and show.
http://study.marearts.com/2015/09/opencv-rtsp-connection-using-vlc-library.html
This method is also good.
But It is cumbersome because it requires additional library file of VLC.

So again, let's solve this problem in OpenCV.
The method is to separately manage the RTSP receving part and drawing part.

New review code.

1. normal method. (This is not stable)
...

#include "opencv2/opencv.hpp"
#include < string>

using namespace std;
using namespace cv;

int main()
{

 string streamUri = "rtsp://192.168.0.21:554/onvif/profile2/media.smp";
 VideoCapture stream(streamUri);
 if (!stream.isOpened()){
  cout << "error" << endl;
  return 0;
 }

 Mat image;
 while (1){
  stream >> image;
  imshow("test", image);
  if (waitKey(30) >= 0)
   break;

 }

 return 0;
}

..

2. Receiving part threading(stable)

...

#include "opencv2/opencv.hpp"
#include < string>
#include < list>
#include < thread>

using namespace std;
using namespace cv;



// RTSP receive buffer list
list< Mat> frames;
cv::VideoCapture stream2;
bool isRun;

// thread function for video getting and show
void StreamThread(bool &isRun)
{
 cv::Mat image;
 while (isRun){
  stream2 >> image;
  frames.push_back(image.clone());
  printf("%d mat stacked \n", frames.size());
 }
}


int main(int, char)
{
 //rtsp address 
 string streamUri = "rtsp://192.168.0.21:554/onvif/profile2/media.smp";
 stream2.open(streamUri);

 //open check
 if (!stream2.isOpened()){
  cerr << "Stream open failed : " << streamUri << endl;
  return EXIT_FAILURE;
 }

 isRun = true;
 // thread run
 thread(StreamThread, isRun).detach();


 //Mat draw only in the main function.
 while (isRun){

  if (frames.size()>1){

   Mat image = frames.front();
   imshow("test", image);
   frames.pop_front();

   if (waitKey(30) >= 0)
    break;
  }
 }

 isRun = false;

 return 0;
}

...

3. Drawing part threading

...

#include "opencv2/opencv.hpp"
#include < string>
#include < list>
#include < thread>

using namespace std;
using namespace cv;



// RTSP receive buffer list
list< Mat> frames;
bool isRun;

// thread function for video show
void drawFrame(bool &isRun)
{
 while (isRun){
  if (frames.size()>1){
   Mat image = frames.front();
   imshow("test", image);
   waitKey(1);
   frames.pop_front();
  }
 }
}


int main(int, char)
{
 //rtsp address 
 string streamUri = "rtsp://192.168.0.21:554/onvif/profile2/media.smp";
 VideoCapture stream(streamUri);

 //open check
 if (!stream.isOpened()){
  cerr << "Stream open failed : " << streamUri << endl;
  return EXIT_FAILURE;
 }

 isRun = true;
 // thread run
 thread(drawFrame, isRun).detach();

 cv::Mat image;
 //Mat get only in the main function.
 while (isRun){

  stream >> image;
  frames.push_back(image.clone());
  printf("%d mat stacked \n", frames.size());
 }

 isRun = false;

 return 0;
}

...

Github
https://github.com/MareArts/OpenCV-RTSP-receiving-Test-in-thread-and-while-processing

Thank you.

opencv rtsp connection using vlc library

RTSP(Real Time Streaming Protocol) is video streaming, it usually sent from network camera.

VideoCapture function in opencv also can get rtsp video, but difficult to receive a stable image.
This is simple example using videocapture.
...

int main()
{

 VideoCapture cap("rtsp://192.168.0.50");

 namedWindow("fish", 0);
 Mat fishEye;
 while (1)
 {
  cap >> fishEye;
  if (fishEye.empty())
   return 0;

  imshow("fish", fishEye);
  if (waitKey(10) > 0)
   break;
 }

 return 0;

}

...

We can receive more reliable RTSP using VLC library.
Of course, there are many different ways(LIVE555, FFmpeg, Qt api...).

#include < windows.h>
#include < vlc/vlc.h>

#include < opencv2\opencv.hpp>    
#include < opencv2\core.hpp>  
#include < opencv2\highgui.hpp>   

#pragma comment(lib, "libvlc.lib")

#ifdef _DEBUG
#pragma comment(lib, "opencv_core300d.lib")  
#pragma comment(lib, "opencv_highgui300d.lib")
#pragma comment(lib, "opencv_imgproc300d.lib")
#pragma comment(lib, "opencv_imgcodecs300d.lib")
#else
#pragma comment(lib, "opencv_core300.lib")  
#pragma comment(lib, "opencv_highgui300.lib")
#pragma comment(lib, "opencv_imgproc300.lib")
#pragma comment(lib, "opencv_imgcodecs300.lib")
#endif

using namespace cv;
using namespace std;

struct ctx
{
 Mat* image;
 HANDLE mutex;
 uchar* pixels;
};
bool isRunning = true;

Size getsize(const char* path)
{
 libvlc_instance_t *vlcInstance;
 libvlc_media_player_t *mp;
 libvlc_media_t *media;

 char *vlc_argv[6];
 vlc_argv[0] = new char[3];
 strcpy_s(vlc_argv[0], 3, "-I");
 vlc_argv[1] = new char[6];
 strcpy_s(vlc_argv[1], 6, "dummy"); // Don't use any interface
 vlc_argv[2] = new char[16];
 strcpy_s(vlc_argv[2], 16, "--ignore-config"); // Don't use VLC's config
 vlc_argv[3] = new char[128];
 strcpy_s(vlc_argv[3], 128, "--plugin-path=/plugins");
 int vlc_argc = 4;

 vlcInstance = libvlc_new(vlc_argc, vlc_argv);

 for (int i = 0; i < vlc_argc; i++)
  delete[] vlc_argv[i];

 media = libvlc_media_new_location(vlcInstance, path);
 mp = libvlc_media_player_new_from_media(media);

 libvlc_media_release(media);
 libvlc_video_set_callbacks(mp, NULL, NULL, NULL, NULL);
 libvlc_video_set_format(mp, "RV24", 100, 100, 100 * 24 / 8); // pitch = width * BitsPerPixel / 8
 //libvlc_video_set_format(mp, "RV32", 100, 100, 100 * 4);
 libvlc_media_player_play(mp);

 Sleep(2000);//wait a while so that something get rendered so that size info is available
 unsigned int width = 640, height = 480;

 for (int i = 0; i < 30 && height == 0; i++)
 {
  Sleep(50);
  libvlc_video_get_size(mp, 0, &width, &height);

  if (width != 0 && height != 0)
   break;
 }


 if (width == 0 || height == 0)
 {
  width = 640;
  height = 480;
 }

 libvlc_media_player_stop(mp);
 libvlc_release(vlcInstance);
 libvlc_media_player_release(mp);
 return Size(width, height);
}


void *lock(void *data, void**p_pixels)
{
 struct ctx *ctx = (struct ctx*)data;
 WaitForSingleObject(ctx->mutex, INFINITE);
 *p_pixels = ctx->pixels;
 return NULL;

}

void display(void *data, void *id){
 (void)data;
 assert(id == NULL);
}

void unlock(void *data, void *id, void *const *p_pixels)
{

 struct ctx *ctx = (struct ctx*)data;
 Mat frame = *ctx->image;
 if (frame.data)
 {
  imshow("frame", frame);
  if (waitKey(1) == 27)
  {
   isRunning = false;
   //exit(0);
  }
 }
 ReleaseMutex(ctx->mutex);
}


int main()
{
 string url = "rtsp://...";
 //vlc sdk does not know the video size until it is rendered, so need to play it a bit so that size is     known
 Size sz = getsize(url.c_str());

 // VLC pointers
 libvlc_instance_t *vlcInstance;
 libvlc_media_player_t *mp;
 libvlc_media_t *media;

 char *vlc_argv[6];
 vlc_argv[0] = new char[3];
 strcpy_s(vlc_argv[0], 3, "-I");
 vlc_argv[1] = new char[6];
 strcpy_s(vlc_argv[1], 6, "dummy"); // Don't use any interface
 vlc_argv[2] = new char[16];
 strcpy_s(vlc_argv[2], 16, "--ignore-config"); // Don't use VLC's config
 vlc_argv[3] = new char[128];
 strcpy_s(vlc_argv[3], 128, "--plugin-path=/plugins");
 int vlc_argc = 4;

 vlcInstance = libvlc_new(vlc_argc, vlc_argv);

 for (int i = 0; i < vlc_argc; i++)
  delete[] vlc_argv[i];

 media = libvlc_media_new_location(vlcInstance, url.c_str());
 mp = libvlc_media_player_new_from_media(media);
 libvlc_media_release(media);


 struct ctx* context = (struct ctx*)malloc(sizeof(*context));
 context->mutex = CreateMutex(NULL, FALSE, NULL);
 context->image = new Mat(sz.height, sz.width, CV_8UC3);
 context->pixels = (unsigned char *)context->image->data;

 libvlc_video_set_callbacks(mp, lock, unlock, display, context);
 libvlc_video_set_format(mp, "RV24", sz.width, sz.height, sz.width * 24 / 8); // pitch = width *     BitsPerPixel / 8
 //libvlc_video_set_format(mp, "RV32", sz.width, sz.height, sz.width * 4);

 libvlc_media_player_play(mp);
 while (isRunning)
 {
  //imshow("rtsp", *(context->image));
  Sleep(1);  
 }

 libvlc_media_player_stop(mp);
 libvlc_release(vlcInstance);
 libvlc_media_player_release(mp);
 free(context);

 return 0;
}

This example code is referenced in here
http://stackoverflow.com/questions/23529620/opencv-and-network-cameras-or-how-to-spy-on-the-neighbors

To use this example code, you should install VLC player, and set dependency referencing include, lib, dll and plugins.
And If you are using opencv 64 bits then you also install 64-bit vlc.

Test video