9/22/2015

opencv rtsp connection using vlc library

RTSP(Real Time Streaming Protocol) is video streaming, it usually sent from network camera.

VideoCapture function in opencv also can get rtsp video, but difficult to receive a stable image.
This is simple example using videocapture.
...
int main()
{

 VideoCapture cap("rtsp://192.168.0.50");

 namedWindow("fish", 0);
 Mat fishEye;
 while (1)
 {
  cap >> fishEye;
  if (fishEye.empty())
   return 0;

  imshow("fish", fishEye);
  if (waitKey(10) > 0)
   break;
 }

 return 0;

}
...

We can receive more reliable RTSP using VLC library.
Of course, there are many different ways(LIVE555, FFmpeg, Qt api...).

#include < windows.h>
#include < vlc/vlc.h>

#include < opencv2\opencv.hpp>    
#include < opencv2\core.hpp>  
#include < opencv2\highgui.hpp>   

#pragma comment(lib, "libvlc.lib")

#ifdef _DEBUG
#pragma comment(lib, "opencv_core300d.lib")  
#pragma comment(lib, "opencv_highgui300d.lib")
#pragma comment(lib, "opencv_imgproc300d.lib")
#pragma comment(lib, "opencv_imgcodecs300d.lib")
#else
#pragma comment(lib, "opencv_core300.lib")  
#pragma comment(lib, "opencv_highgui300.lib")
#pragma comment(lib, "opencv_imgproc300.lib")
#pragma comment(lib, "opencv_imgcodecs300.lib")
#endif

using namespace cv;
using namespace std;

struct ctx
{
 Mat* image;
 HANDLE mutex;
 uchar* pixels;
};
bool isRunning = true;

Size getsize(const char* path)
{
 libvlc_instance_t *vlcInstance;
 libvlc_media_player_t *mp;
 libvlc_media_t *media;

 char *vlc_argv[6];
 vlc_argv[0] = new char[3];
 strcpy_s(vlc_argv[0], 3, "-I");
 vlc_argv[1] = new char[6];
 strcpy_s(vlc_argv[1], 6, "dummy"); // Don't use any interface
 vlc_argv[2] = new char[16];
 strcpy_s(vlc_argv[2], 16, "--ignore-config"); // Don't use VLC's config
 vlc_argv[3] = new char[128];
 strcpy_s(vlc_argv[3], 128, "--plugin-path=/plugins");
 int vlc_argc = 4;

 vlcInstance = libvlc_new(vlc_argc, vlc_argv);

 for (int i = 0; i < vlc_argc; i++)
  delete[] vlc_argv[i];

 media = libvlc_media_new_location(vlcInstance, path);
 mp = libvlc_media_player_new_from_media(media);

 libvlc_media_release(media);
 libvlc_video_set_callbacks(mp, NULL, NULL, NULL, NULL);
 libvlc_video_set_format(mp, "RV24", 100, 100, 100 * 24 / 8); // pitch = width * BitsPerPixel / 8
 //libvlc_video_set_format(mp, "RV32", 100, 100, 100 * 4);
 libvlc_media_player_play(mp);

 Sleep(2000);//wait a while so that something get rendered so that size info is available
 unsigned int width = 640, height = 480;

 for (int i = 0; i < 30 && height == 0; i++)
 {
  Sleep(50);
  libvlc_video_get_size(mp, 0, &width, &height);

  if (width != 0 && height != 0)
   break;
 }


 if (width == 0 || height == 0)
 {
  width = 640;
  height = 480;
 }

 libvlc_media_player_stop(mp);
 libvlc_release(vlcInstance);
 libvlc_media_player_release(mp);
 return Size(width, height);
}


void *lock(void *data, void**p_pixels)
{
 struct ctx *ctx = (struct ctx*)data;
 WaitForSingleObject(ctx->mutex, INFINITE);
 *p_pixels = ctx->pixels;
 return NULL;

}

void display(void *data, void *id){
 (void)data;
 assert(id == NULL);
}

void unlock(void *data, void *id, void *const *p_pixels)
{

 struct ctx *ctx = (struct ctx*)data;
 Mat frame = *ctx->image;
 if (frame.data)
 {
  imshow("frame", frame);
  if (waitKey(1) == 27)
  {
   isRunning = false;
   //exit(0);
  }
 }
 ReleaseMutex(ctx->mutex);
}


int main()
{
 string url = "rtsp://...";
 //vlc sdk does not know the video size until it is rendered, so need to play it a bit so that size is     known
 Size sz = getsize(url.c_str());

 // VLC pointers
 libvlc_instance_t *vlcInstance;
 libvlc_media_player_t *mp;
 libvlc_media_t *media;

 char *vlc_argv[6];
 vlc_argv[0] = new char[3];
 strcpy_s(vlc_argv[0], 3, "-I");
 vlc_argv[1] = new char[6];
 strcpy_s(vlc_argv[1], 6, "dummy"); // Don't use any interface
 vlc_argv[2] = new char[16];
 strcpy_s(vlc_argv[2], 16, "--ignore-config"); // Don't use VLC's config
 vlc_argv[3] = new char[128];
 strcpy_s(vlc_argv[3], 128, "--plugin-path=/plugins");
 int vlc_argc = 4;

 vlcInstance = libvlc_new(vlc_argc, vlc_argv);

 for (int i = 0; i < vlc_argc; i++)
  delete[] vlc_argv[i];

 media = libvlc_media_new_location(vlcInstance, url.c_str());
 mp = libvlc_media_player_new_from_media(media);
 libvlc_media_release(media);


 struct ctx* context = (struct ctx*)malloc(sizeof(*context));
 context->mutex = CreateMutex(NULL, FALSE, NULL);
 context->image = new Mat(sz.height, sz.width, CV_8UC3);
 context->pixels = (unsigned char *)context->image->data;

 libvlc_video_set_callbacks(mp, lock, unlock, display, context);
 libvlc_video_set_format(mp, "RV24", sz.width, sz.height, sz.width * 24 / 8); // pitch = width *     BitsPerPixel / 8
 //libvlc_video_set_format(mp, "RV32", sz.width, sz.height, sz.width * 4);

 libvlc_media_player_play(mp);
 while (isRunning)
 {
  //imshow("rtsp", *(context->image));
  Sleep(1);  
 }

 libvlc_media_player_stop(mp);
 libvlc_release(vlcInstance);
 libvlc_media_player_release(mp);
 free(context);

 return 0;
}


This example code is referenced in here
http://stackoverflow.com/questions/23529620/opencv-and-network-cameras-or-how-to-spy-on-the-neighbors

To use this example code, you should install VLC player, and set dependency referencing include, lib, dll and plugins.
And If you are using opencv 64 bits then you also install 64-bit vlc.

Test video


20 comments:

  1. Thank you for this code. Is there any chance to compare two frames ?
    I mean something like this:
    cap.read(frame1);
    cap.read(frame2);
    ..
    but this is without libVlc.

    Thank you for helpful informations

    ReplyDelete
    Replies
    1. If you make Mat buffer ex) Vector< Mat >, it will be possible.
      Or refer to this site http://study.marearts.com/2016/03/opencv-rtsp-receiving-test.html. no use vlc, so we can receive rtsp more easily.
      Thank you.

      Delete
    2. When I try to use some simply method do get video from RTSP, I get just blured frames, and I dont know why. Can be problem in network ?
      But when I use libVlc, the blur not happens.
      Is there any chance to save the two frames to Matrixs, with using libVlc ?

      Delete
    3. You right.
      Could you give me a little time(about 1week)?, I will make simple code.
      Thank you.

      Delete
  2. It is OK, thak you very much for your time

    ReplyDelete
    Replies
    1. You're welcome.
      Thank you.

      Delete
    2. Do you work on it ? I don't know if we understood each other :)

      Delete
    3. Yes, we dis-understand each other.
      But now I don't have time.
      My job is very busy.
      To answer seems to take a lot of time.
      Sorry.

      Delete
    4. It is OK, I change the bit rate and resolution of camera, it helps me, but i lost quality of stream..

      Delete
  3. thanks fort the code
    can i use this code for stream my xioami yi action cam?
    because xiaomi yi action use rtsp
    thanks a lot :)

    ReplyDelete
    Replies
    1. No matter the type of camera, it seems to be run, if standard rtsp.
      Thank you.

      Delete
  4. opencv3.1.0 or opencv 3.0.0

    ReplyDelete
  5. This comment has been removed by the author.

    ReplyDelete
  6. Are you afraid how to operate VLC player. It is very easy to deal with howtodoninja.com You are exempted from downloading Codec or any other specially designed software tool for the activation of VLC player.

    ReplyDelete
  7. If you think that you can only play DVDs on your VLC player then check your facts again. You can play an array of other physical video storage devices upon this player including CD, VCD and even SVCD. VLC Media Player Guides

    ReplyDelete
  8. Wow, cool post. I'd like to write like this too - taking time and real hard work to make a great article... but I put things off too much and never seem to get started. Thanks though. cfa books pdf

    ReplyDelete
  9. I have read all the comments and suggestions posted by the visitors for this article are very fine,We will wait for your next article so only.Thanks! cfa level 1 study material

    ReplyDelete
  10. Took me time to read all the comments, but I really enjoyed the article. It proved to be Very helpful to me and I am sure to all the commenters here! It’s always nice when you can not only be informed, but also entertained! cfa level 1 summaries

    ReplyDelete
  11. Very useful post. This is my first time i visit here. I found so many interesting stuff in your blog especially its discussion. Really its great article. Keep it up. commercial audio visual solution

    ReplyDelete
  12. Love to read it,Waiting For More new Update and I Already Read your Recent Post its Great Thanks. buy toner

    ReplyDelete