Installing OpenCV 3.2.0 with contrib modules in Ubuntu 16.04 LTS

UPDATE: You can also install OpenCV 4.5.0 in Ubuntu 20.04LTS.

OpenCV 3.2.0 has been out for a while and contains many improvements and exciting new features, so it’s time to update this guide using the latest Ubuntu 16.04LTS.

A big change in OpenCV 3.2.0 is that now many of the newest algorithms now reside separately in the contrib repository.

Some of these modules include Face Recognition, RGB-Depth processing, Image Registration, Saliency, Structure From Motion, Tracking, and much more.

So let’s install OpenCV 3.2.0 with the contrib modules and other good stuff by executing the following code on the command line:

sudo apt-get update
sudo apt-get upgrade
sudo apt-get install build-essential libgtk2.0-dev libjpeg-dev libtiff5-dev libjasper-dev libopenexr-dev cmake python-dev python-numpy python-tk libtbb-dev libeigen3-dev yasm libfaac-dev libopencore-amrnb-dev libopencore-amrwb-dev libtheora-dev libvorbis-dev libxvidcore-dev libx264-dev libqt4-dev libqt4-opengl-dev sphinx-common texlive-latex-extra libv4l-dev libdc1394-22-dev libavcodec-dev libavformat-dev libswscale-dev default-jdk ant libvtk5-qt4-dev
cd ~
mkdir opencv
cd opencv
wget https://github.com/opencv/opencv/archive/3.2.0.tar.gz
tar -xvzf 3.2.0.tar.gz
wget https://github.com/opencv/opencv_contrib/archive/3.2.0.zip
unzip 3.2.0.zip
cd opencv-3.2.0

But before we build it, we need to fix one problem currently present in the contrib modules, specifically in the freetype module, which allows you to draw UTF-8 strings. If you are getting an error similar to ImportError: /usr/local/lib/libopencv_freetype.so.3.2: undefined symbol: hb_shape, this will fix it:

sed -i 's/${freetype2_LIBRARIES} ${harfbuzz_LIBRARIES}/${FREETYPE_LIBRARIES} ${HARFBUZZ_LIBRARIES}/g' ../opencv_contrib-3.2.0/modules/freetype/CMakeLists.txt

Since now that problem is solved, now we are ready to build OpenCV.

mkdir build
cd build
cmake -D WITH_TBB=ON -D BUILD_NEW_PYTHON_SUPPORT=ON -D WITH_V4L=ON -D INSTALL_C_EXAMPLES=ON -D INSTALL_PYTHON_EXAMPLES=ON -D BUILD_EXAMPLES=ON -D WITH_QT=ON -D WITH_OPENGL=ON -D WITH_VTK=ON .. -DCMAKE_BUILD_TYPE=RELEASE -DOPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-3.2.0/modules ..
make
sudo make install
echo '/usr/local/lib' | sudo tee --append /etc/ld.so.conf.d/opencv.conf
sudo ldconfig
echo 'PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig' | sudo tee --append ~/.bashrc
echo 'export PKG_CONFIG_PATH' | sudo tee --append ~/.bashrc
source ~/.bashrc

Now you should be able to compile with the OpenCV libraries, including the contrib repositories.

For example, let’s compute some Fine-Grained Saliency, which is available in the saliency module of the contrib repository:

cd ~
mkdir saliency
cd saliency
cp ../opencv/opencv_contrib-3.2.0/modules/saliency/samples/computeSaliency.cpp .
cp ../opencv/opencv-3.2.0/samples/data/Megamind.avi .
g++ -o computeSaliency `pkg-config opencv --cflags` computeSaliency.cpp `pkg-config opencv --libs`
./computeSaliency FINE_GRAINED Megamind.avi 23

Original Image:

Computed Saliency:

Posted in Open Source, OpenCV, Programming.

rev="post-423" 17 comments

By samontab – June 3, 2017

Interfacing Intel RealSense F200 with OpenCV

RealSense from Intel is an interesting technology that brings 3D cameras into the mainstream.
Although the RealSense SDK provides a good starting point for many applications, some users would prefer to have a bit more control over the images. In this post, I’ll describe how to access the raw streams from an Intel RealSense F200 camera, and how to convert those images into OpenCV cv::Mat objects.

Here is the video with all the explanations

And here are the files used in the video:

01-alignedSingleThreaded.cpp:

#include &lt;pxcsensemanager.h&gt;
#include &lt;opencv2/opencv.hpp&gt;

PXCSenseManager *pxcSenseManager;

PXCImage * CVMat2PXCImage(cv::Mat cvImage)
{
    PXCImage::ImageInfo iinfo;
    memset(&amp;iinfo,0,sizeof(iinfo));
    iinfo.width=cvImage.cols;
    iinfo.height=cvImage.rows;

    PXCImage::PixelFormat format;
    int type = cvImage.type();
    if(type == CV_8UC1)
        format = PXCImage::PIXEL_FORMAT_Y8;
    else if(type == CV_8UC3)
        format = PXCImage::PIXEL_FORMAT_RGB24;
    else if(type == CV_32FC1)
        format = PXCImage::PIXEL_FORMAT_DEPTH_F32;

    iinfo.format = format;

    PXCImage *pxcImage = pxcSenseManager-&gt;QuerySession()-&gt;CreateImage(&amp;iinfo);

    PXCImage::ImageData data;
    pxcImage-&gt;AcquireAccess(PXCImage::ACCESS_WRITE, format, &amp;data);

    data.planes[0] = cvImage.data;

    pxcImage-&gt;ReleaseAccess(&amp;data);
    return pxcImage;
}

cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage-&gt;AcquireAccess(PXCImage::ACCESS_READ, format, &amp;data);

    int width = pxcImage-&gt;QueryInfo().width;
    int height = pxcImage-&gt;QueryInfo().height;
    if(!format)
        format = pxcImage-&gt;QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH)
        type = CV_16UC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage-&gt;ReleaseAccess(&amp;data);
    return ocvImage;
}

int main(int argc, char* argv[])
{
    //Define some parameters for the camera
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    //Create the OpenCV windows and images
    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    cv::Mat frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    cv::Mat frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    cv::Mat frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);

    //Initialize the RealSense Manager
    pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_IR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_COLOR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_DEPTH, frameSize.width, frameSize.height, frameRate);

    //Initialize the pipeline
    pxcSenseManager-&gt;Init();

    bool keepRunning = true;
    while(keepRunning)
    {
        //Acquire all the frames from the camera
        pxcSenseManager-&gt;AcquireFrame();
        PXCCapture::Sample *sample = pxcSenseManager-&gt;QuerySample();

        //Convert each frame into an OpenCV image
        frameIR = PXCImage2CVMat(sample-&gt;ir, PXCImage::PIXEL_FORMAT_Y8);
        frameColor = PXCImage2CVMat(sample-&gt;color, PXCImage::PIXEL_FORMAT_RGB24);
        cv::Mat frameDepth_u16 = PXCImage2CVMat(sample-&gt;depth, PXCImage::PIXEL_FORMAT_DEPTH);
        frameDepth_u16.convertTo(frameDepth, CV_8UC1);

        cv::Mat frameDisplay;
        cv::equalizeHist(frameDepth, frameDisplay);

        //Display the images
        cv::imshow("IR", frameIR);
        cv::imshow("Color", frameColor);
        cv::imshow("Depth", frameDisplay);

        //Check for user input
        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;

        //Release the memory from the frames
        pxcSenseManager-&gt;ReleaseFrame();
    }

    //Release the memory from the RealSense manager
    pxcSenseManager-&gt;Release();

    return 0;
}
02-unalignedSingleThreaded.cpp:
#include &lt;pxcsensemanager.h&gt;
#include &lt;opencv2/opencv.hpp&gt;

cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage-&gt;AcquireAccess(PXCImage::ACCESS_READ, format, &amp;data);

    int width = pxcImage-&gt;QueryInfo().width;
    int height = pxcImage-&gt;QueryInfo().height;

    if(!format)
        format = pxcImage-&gt;QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage-&gt;ReleaseAccess(&amp;data);
    return ocvImage;
}

int main(int argc, char* argv[])
{
    //Define some parameters for the camera
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    //Create the OpenCV windows and images
    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    cv::Mat frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    cv::Mat frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    cv::Mat frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);

    //Initialize the RealSense Manager
    PXCSenseManager *pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_IR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_COLOR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_DEPTH, frameSize.width, frameSize.height, frameRate);

    //Initialize the pipeline
    pxcSenseManager-&gt;Init();

    bool keepRunning = true;
    while(keepRunning)
    {
        //Acquire any frame from the camera
        pxcSenseManager-&gt;AcquireFrame(false);
        PXCCapture::Sample *sample = pxcSenseManager-&gt;QuerySample();

        //Convert each frame into an OpenCV image
        //You need to make sure that the image is there first
        if(sample-&gt;ir)
            frameIR = PXCImage2CVMat(sample-&gt;ir, PXCImage::PIXEL_FORMAT_Y8);
        if(sample-&gt;color)
            frameColor = PXCImage2CVMat(sample-&gt;color, PXCImage::PIXEL_FORMAT_RGB24);
        if(sample-&gt;depth)
            PXCImage2CVMat(sample-&gt;depth, PXCImage::PIXEL_FORMAT_DEPTH_F32).convertTo(frameDepth, CV_8UC1);

        //Display the images
        cv::imshow("IR", frameIR);
        cv::imshow("Color", frameColor);
        cv::imshow("Depth", frameDepth);

        //Check for user input
        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;

        //Release the memory from the frames
        pxcSenseManager-&gt;ReleaseFrame();
    }

    //Release the memory from the RealSense manager
    pxcSenseManager-&gt;Release();

    return 0;
}
03-unalignedMultiThreaded.cpp:
#include &lt;pxcsensemanager.h&gt;
#include &lt;iostream&gt;
#include &lt;opencv2/opencv.hpp&gt;

cv::Mat frameIR;
cv::Mat frameColor;
cv::Mat frameDepth;
cv::Mutex framesMutex;

cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage-&gt;AcquireAccess(PXCImage::ACCESS_READ, format, &amp;data);

    int width = pxcImage-&gt;QueryInfo().width;
    int height = pxcImage-&gt;QueryInfo().height;

    if(!format)
        format = pxcImage-&gt;QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage-&gt;ReleaseAccess(&amp;data);
    return ocvImage;
}

class FramesHandler:public PXCSenseManager::Handler
{
public:
    virtual pxcStatus PXCAPI OnNewSample(pxcUID, PXCCapture::Sample *sample)
    {
            framesMutex.lock();
                if(sample-&gt;ir)
                    frameIR = PXCImage2CVMat(sample-&gt;ir, PXCImage::PIXEL_FORMAT_Y8);
                if(sample-&gt;color)
                    frameColor = PXCImage2CVMat(sample-&gt;color, PXCImage::PIXEL_FORMAT_RGB24);
                if(sample-&gt;depth)
                    PXCImage2CVMat(sample-&gt;depth, PXCImage::PIXEL_FORMAT_DEPTH_F32).convertTo(frameDepth, CV_8UC1);
            framesMutex.unlock();
    return PXC_STATUS_NO_ERROR;
    }
};

int main(int argc, char* argv[])
{
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);

    PXCSenseManager *pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_IR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_COLOR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager-&gt;EnableStream(PXCCapture::STREAM_TYPE_DEPTH, frameSize.width, frameSize.height, frameRate);

    FramesHandler handler;
    pxcSenseManager-&gt;Init(&amp;handler);
    pxcSenseManager-&gt;StreamFrames(false);

    //Local images for display
    cv::Mat displayIR = frameIR.clone();
    cv::Mat displayColor = frameColor.clone();
    cv::Mat displayDepth = frameDepth.clone();

    bool keepRunning = true;
    while(keepRunning)
    {
        framesMutex.lock();
            displayIR = frameIR.clone();
            displayColor = frameColor.clone();
            displayDepth = frameDepth.clone();
        framesMutex.unlock();

        cv::imshow("IR", displayIR);
        cv::imshow("Color", displayColor);
        cv::imshow("Depth", displayDepth);

        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;
    }
    //Stop the frame acqusition thread
    pxcSenseManager-&gt;Close();

    pxcSenseManager-&gt;Release();

    return 0;
}
04-alignedMultiThreaded.cpp:
#include &lt;pxcsensemanager.h&gt;
#include &lt;iostream&gt;
#include &lt;opencv2/opencv.hpp&gt;

cv::Mat frameIR;
cv::Mat frameColor;
cv::Mat frameDepth;
cv::Mutex framesMutex;

cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage-&gt;AcquireAccess(PXCImage::ACCESS_READ, format, &amp;data);

    int width = pxcImage-&gt;QueryInfo().width;
    int height = pxcImage-&gt;QueryInfo().height;

    if(!format)
        format = pxcImage-&gt;QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage-&gt;ReleaseAccess(&amp;data);
    return ocvImage;
}

class FramesHandler:public PXCSenseManager::Handler
{
public:
    virtual pxcStatus PXCAPI OnNewSample(pxcUID, PXCCapture::Sample *sample)
    {
            framesMutex.lock();
                frameIR = PXCImage2CVMat(sample-&gt;ir, PXCImage::PIXEL_FORMAT_Y8);
                frameColor = PXCImage2CVMat(sample-&gt;color, PXCImage::PIXEL_FORMAT_RGB24);
                PXCImage2CVMat(sample-&gt;depth, PXCImage::PIXEL_FORMAT_DEPTH_F32).convertTo(frameDepth, CV_8UC1);
            framesMutex.unlock();
    return PXC_STATUS_NO_ERROR;
    }
};

int main(int argc, char* argv[])
{
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);

    PXCSenseManager *pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    PXCVideoModule::DataDesc ddesc={};
    ddesc.deviceInfo.streams = PXCCapture::STREAM_TYPE_IR | PXCCapture::STREAM_TYPE_COLOR | PXCCapture::STREAM_TYPE_DEPTH;

    pxcSenseManager-&gt;EnableStreams(&amp;ddesc);

    FramesHandler handler;
    pxcSenseManager-&gt;Init(&amp;handler);
    pxcSenseManager-&gt;StreamFrames(false);

    //Local images for display
    cv::Mat displayIR = frameIR.clone();
    cv::Mat displayColor = frameColor.clone();
    cv::Mat displayDepth = frameDepth.clone();

    bool keepRunning = true;
    while(keepRunning)
    {
        framesMutex.lock();
            displayIR = frameIR.clone();
            displayColor = frameColor.clone();
            displayDepth = frameDepth.clone();
        framesMutex.unlock();

        cv::imshow("IR", displayIR);
        cv::imshow("Color", displayColor);
        cv::imshow("Depth", displayDepth);

        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;
    }
    //Stop the frame acqusition thread
    pxcSenseManager-&gt;Close();

    pxcSenseManager-&gt;Release();

    return 0;
}

Posted in Computer Vision, IoT, OpenCV, Photography, Programming.