Skip to content


Interfacing Intel RealSense F200 with OpenCV

RealSense from Intel is an interesting technology that brings 3D cameras into the mainstream.
Although the RealSense SDK provides a good starting point for many applications, some users would prefer to have a bit more control over the images. In this post, I’ll describe how to access the raw streams from an Intel RealSense F200 camera, and how to convert those images into OpenCV cv::Mat objects.

Here is the video with all the explanations:

And here are the files used in the video:

01-alignedSingleThreaded.cpp:

#include <pxcsensemanager.h>
#include <opencv2/opencv.hpp>

PXCSenseManager *pxcSenseManager;


PXCImage * CVMat2PXCImage(cv::Mat cvImage)
{
    PXCImage::ImageInfo iinfo;
    memset(&iinfo,0,sizeof(iinfo));
    iinfo.width=cvImage.cols;
    iinfo.height=cvImage.rows;

    PXCImage::PixelFormat format;
    int type = cvImage.type();
    if(type == CV_8UC1)
        format = PXCImage::PIXEL_FORMAT_Y8;
    else if(type == CV_8UC3)
        format = PXCImage::PIXEL_FORMAT_RGB24;
    else if(type == CV_32FC1)
        format = PXCImage::PIXEL_FORMAT_DEPTH_F32;

    iinfo.format = format;


    PXCImage *pxcImage = pxcSenseManager->QuerySession()->CreateImage(&iinfo);

    PXCImage::ImageData data;
    pxcImage->AcquireAccess(PXCImage::ACCESS_WRITE, format, &data);

    data.planes[0] = cvImage.data;

    pxcImage->ReleaseAccess(&data);
    return pxcImage;
}


cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage->AcquireAccess(PXCImage::ACCESS_READ, format, &data);

    int width = pxcImage->QueryInfo().width;
    int height = pxcImage->QueryInfo().height;
    if(!format)
        format = pxcImage->QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH)
        type = CV_16UC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);


    pxcImage->ReleaseAccess(&data);
    return ocvImage;
}

int main(int argc, char* argv[])
{
    //Define some parameters for the camera
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    //Create the OpenCV windows and images
    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    cv::Mat frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    cv::Mat frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    cv::Mat frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);


    //Initialize the RealSense Manager
    pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_IR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_COLOR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_DEPTH, frameSize.width, frameSize.height, frameRate);

    //Initialize the pipeline
    pxcSenseManager->Init();

    bool keepRunning = true;
    while(keepRunning)
    {
        //Acquire all the frames from the camera
        pxcSenseManager->AcquireFrame();
        PXCCapture::Sample *sample = pxcSenseManager->QuerySample();

        //Convert each frame into an OpenCV image
        frameIR = PXCImage2CVMat(sample->ir, PXCImage::PIXEL_FORMAT_Y8);
        frameColor = PXCImage2CVMat(sample->color, PXCImage::PIXEL_FORMAT_RGB24);
        cv::Mat frameDepth_u16 = PXCImage2CVMat(sample->depth, PXCImage::PIXEL_FORMAT_DEPTH);
        frameDepth_u16.convertTo(frameDepth, CV_8UC1);

        cv::Mat frameDisplay;
        cv::equalizeHist(frameDepth, frameDisplay);

        //Display the images
        cv::imshow("IR", frameIR);
        cv::imshow("Color", frameColor);
        cv::imshow("Depth", frameDisplay);

        //Check for user input
        int key = cv::waitKey(1);
        if(key == 27)            
            keepRunning = false;

        //Release the memory from the frames
        pxcSenseManager->ReleaseFrame();
    }

    //Release the memory from the RealSense manager
    pxcSenseManager->Release();

    return 0;
}

02-unalignedSingleThreaded.cpp:

#include <pxcsensemanager.h>
#include <opencv2/opencv.hpp>

cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage->AcquireAccess(PXCImage::ACCESS_READ, format, &data);

    int width = pxcImage->QueryInfo().width;
    int height = pxcImage->QueryInfo().height;

    if(!format)
        format = pxcImage->QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage->ReleaseAccess(&data);
    return ocvImage;
}

int main(int argc, char* argv[])
{
    //Define some parameters for the camera
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    //Create the OpenCV windows and images
    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    cv::Mat frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    cv::Mat frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    cv::Mat frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);


    //Initialize the RealSense Manager
    PXCSenseManager *pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_IR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_COLOR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_DEPTH, frameSize.width, frameSize.height, frameRate);

    //Initialize the pipeline
    pxcSenseManager->Init();

    bool keepRunning = true;
    while(keepRunning)
    {
        //Acquire any frame from the camera
        pxcSenseManager->AcquireFrame(false);
        PXCCapture::Sample *sample = pxcSenseManager->QuerySample();

        //Convert each frame into an OpenCV image
        //You need to make sure that the image is there first
        if(sample->ir)
            frameIR = PXCImage2CVMat(sample->ir, PXCImage::PIXEL_FORMAT_Y8);
        if(sample->color)
            frameColor = PXCImage2CVMat(sample->color, PXCImage::PIXEL_FORMAT_RGB24);
        if(sample->depth)
            PXCImage2CVMat(sample->depth, PXCImage::PIXEL_FORMAT_DEPTH_F32).convertTo(frameDepth, CV_8UC1);

        //Display the images
        cv::imshow("IR", frameIR);
        cv::imshow("Color", frameColor);
        cv::imshow("Depth", frameDepth);

        //Check for user input
        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;

        //Release the memory from the frames
        pxcSenseManager->ReleaseFrame();
    }

    //Release the memory from the RealSense manager
    pxcSenseManager->Release();

    return 0;
}

03-unalignedMultiThreaded.cpp:

#include <pxcsensemanager.h>
#include <iostream>
#include <opencv2/opencv.hpp>

cv::Mat frameIR;
cv::Mat frameColor;
cv::Mat frameDepth;
cv::Mutex framesMutex;


cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage->AcquireAccess(PXCImage::ACCESS_READ, format, &data);

    int width = pxcImage->QueryInfo().width;
    int height = pxcImage->QueryInfo().height;

    if(!format)
        format = pxcImage->QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage->ReleaseAccess(&data);
    return ocvImage;
}



class FramesHandler:public PXCSenseManager::Handler
{
public:
    virtual pxcStatus PXCAPI OnNewSample(pxcUID, PXCCapture::Sample *sample)
    {
            framesMutex.lock();
                if(sample->ir)
                    frameIR = PXCImage2CVMat(sample->ir, PXCImage::PIXEL_FORMAT_Y8);
                if(sample->color)
                    frameColor = PXCImage2CVMat(sample->color, PXCImage::PIXEL_FORMAT_RGB24);
                if(sample->depth)
                    PXCImage2CVMat(sample->depth, PXCImage::PIXEL_FORMAT_DEPTH_F32).convertTo(frameDepth, CV_8UC1);
            framesMutex.unlock();
    return PXC_STATUS_NO_ERROR;
    }
};


int main(int argc, char* argv[])
{
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);

    PXCSenseManager *pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_IR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_COLOR, frameSize.width, frameSize.height, frameRate);
    pxcSenseManager->EnableStream(PXCCapture::STREAM_TYPE_DEPTH, frameSize.width, frameSize.height, frameRate);

    FramesHandler handler;
    pxcSenseManager->Init(&handler);
    pxcSenseManager->StreamFrames(false);

    //Local images for display
    cv::Mat displayIR = frameIR.clone();
    cv::Mat displayColor = frameColor.clone();
    cv::Mat displayDepth = frameDepth.clone();

    bool keepRunning = true;
    while(keepRunning)
    {
        framesMutex.lock();
            displayIR = frameIR.clone();
            displayColor = frameColor.clone();
            displayDepth = frameDepth.clone();
        framesMutex.unlock();

        cv::imshow("IR", displayIR);
        cv::imshow("Color", displayColor);
        cv::imshow("Depth", displayDepth);

        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;
    }
    //Stop the frame acqusition thread
    pxcSenseManager->Close();

    pxcSenseManager->Release();


    return 0;
}

04-alignedMultiThreaded.cpp:

#include <pxcsensemanager.h>
#include <iostream>
#include <opencv2/opencv.hpp>

cv::Mat frameIR;
cv::Mat frameColor;
cv::Mat frameDepth;
cv::Mutex framesMutex;


cv::Mat PXCImage2CVMat(PXCImage *pxcImage, PXCImage::PixelFormat format)
{
    PXCImage::ImageData data;
    pxcImage->AcquireAccess(PXCImage::ACCESS_READ, format, &data);

    int width = pxcImage->QueryInfo().width;
    int height = pxcImage->QueryInfo().height;

    if(!format)
        format = pxcImage->QueryInfo().format;

    int type;
    if(format == PXCImage::PIXEL_FORMAT_Y8)
        type = CV_8UC1;
    else if(format == PXCImage::PIXEL_FORMAT_RGB24)
        type = CV_8UC3;
    else if(format == PXCImage::PIXEL_FORMAT_DEPTH_F32)
        type = CV_32FC1;

    cv::Mat ocvImage = cv::Mat(cv::Size(width, height), type, data.planes[0]);

    pxcImage->ReleaseAccess(&data);
    return ocvImage;
}



class FramesHandler:public PXCSenseManager::Handler
{
public:
    virtual pxcStatus PXCAPI OnNewSample(pxcUID, PXCCapture::Sample *sample)
    {
            framesMutex.lock();
                frameIR = PXCImage2CVMat(sample->ir, PXCImage::PIXEL_FORMAT_Y8);
                frameColor = PXCImage2CVMat(sample->color, PXCImage::PIXEL_FORMAT_RGB24);
                PXCImage2CVMat(sample->depth, PXCImage::PIXEL_FORMAT_DEPTH_F32).convertTo(frameDepth, CV_8UC1);
            framesMutex.unlock();
    return PXC_STATUS_NO_ERROR;
    }
};


int main(int argc, char* argv[])
{
    cv::Size frameSize = cv::Size(640, 480);
    float frameRate = 60;

    cv::namedWindow("IR", cv::WINDOW_NORMAL);
    cv::namedWindow("Color", cv::WINDOW_NORMAL);
    cv::namedWindow("Depth", cv::WINDOW_NORMAL);
    frameIR = cv::Mat::zeros(frameSize, CV_8UC1);
    frameColor = cv::Mat::zeros(frameSize, CV_8UC3);
    frameDepth = cv::Mat::zeros(frameSize, CV_8UC1);

    PXCSenseManager *pxcSenseManager = PXCSenseManager::CreateInstance();

    //Enable the streams to be used
    PXCVideoModule::DataDesc ddesc={};
    ddesc.deviceInfo.streams = PXCCapture::STREAM_TYPE_IR | PXCCapture::STREAM_TYPE_COLOR | PXCCapture::STREAM_TYPE_DEPTH;

    pxcSenseManager->EnableStreams(&ddesc);

    FramesHandler handler;
    pxcSenseManager->Init(&handler);
    pxcSenseManager->StreamFrames(false);

    //Local images for display
    cv::Mat displayIR = frameIR.clone();
    cv::Mat displayColor = frameColor.clone();
    cv::Mat displayDepth = frameDepth.clone();

    bool keepRunning = true;
    while(keepRunning)
    {
        framesMutex.lock();
            displayIR = frameIR.clone();
            displayColor = frameColor.clone();
            displayDepth = frameDepth.clone();
        framesMutex.unlock();

        cv::imshow("IR", displayIR);
        cv::imshow("Color", displayColor);
        cv::imshow("Depth", displayDepth);

        int key = cv::waitKey(1);
        if(key == 27)
            keepRunning = false;       
    }
    //Stop the frame acqusition thread
    pxcSenseManager->Close();

    pxcSenseManager->Release();


    return 0;
}

Posted in 3D, Image Processing, Photography, Programming.


Understanding how virtual make-up apps work

Virtual make-up is an interesting technology that can be used to aid in the decision of buying cosmetics, enhancing portraits, or just for fun.

Since the final color of an applied cosmetic depends both, on the color of the cosmetic, and on skin color, most of the time people have to go to the stores and try the products on themselves to see how they would look like. With virtual make-up technology, this can be conveniently simulated on a computer, or a mobile phone. The only thing needed is a photograph of their face looking towards the camera, and the software is able to simulate how a particular make-up would look on that particular skin color. This can also be applied to a live camera feed for a more realistic application with real time rendering.

To create an application like this, the first step is to estimate where the faces are in the photograph. This can be solved with computer vision. In the general case, this problem is called object detection. You need to define how your object looks like, and then train an algorithm with many images of the object. Most computer vision algorithms that perform this task assume that the face appears on a roughly frontal view, with little or no objects covering it. This is because most faces have a similar structure when viewed from the front, whereas profile or back views of the head change a lot from person to person because of hair styles, among other things.

In order to capture the facial structure, these algorithms are usually trained with features such as Haar-like, lbp, or HoG. Once trained, the algorithm is able to detect faces in images.

So, for example, let’s say that you start with an image like this:

0

After detecting the face, you will end up with an region of interest on the image. Something like this:

girl.jpg_faceBorder

Now, inside that region of interest, we need to detect specific face landmarks. These landmarks represent the position of different parts of the face, such as eyes, mouth, and eyebrows. This is again, an object detection problem. You need to train an algorithm with many annotated images and a specific number of facial landmarks. Then, you can use this trained algorithm to detect those facial features in a new image. Just like this for example:

girlPoints

Once you have the position of those landmarks, you need to design your own make-up, and align it to those features. Once you have that, you can then blend together the original image with your designed make-up. Here are some basic examples with a few different colors:

1  3 24

The position of the detected landmarks and the design of the make-up are crucial to make it appear realistic. On top of that, there are many different computer vision techniques that can be applied in order to blend the make-up into the face in a more realistic manner.

Posted in General, Image Processing, Photography, Programming.


Interfacing Intel RealSense 3D camera with Google Camera’s Lens Blur

There is an interesting mode in the Google Camera app, called Lens Blur, that allows you to refocus a picture after it was taken, basically simulating what a light-field camera is able to do but using just a regular smartphone.

To do this, the app uses an array of computer vision techniques such as Structure-from-Motion(SfM), and Multi-View-Stereo(MVS) to create a depth map of the scene. Having this entire pipeline running on a smartphone is remarkable to say the least. You can read more about how this is done here.

Once the depth map and the photo are acquired, the user can select the focus location, and the desired depth of field. A thin lens model is then used to simulate a real lens matching the desired focus location and depth of field, generating a new image.

In theory, any photograph with its corresponding depth map could be used with this technique. To test this, I decided to use an Intel RealSense F200 camera, and it worked. Here are the results:

Focused on the front:
frontInFocus

Focused on the back:
backInFocus

Those two images were created on the smartphone using the Lens Blur feature of the Google Camera app. The input image was created by me externally, but the app was happy to process it since I used the same encoding that the app uses.

To do that, I first took a color and depth image from the Realsense camera. Then, projected the depth image into the color camera:

Color photograph from RealSense F200:
realsense_color

Projected depth image from RealSense F200:
realsense_depth

The next step is to encode the depth image into a format that Google Camera understands, so I followed the encoding instructions from the documentation. The RangeLinear encoding of the previous depth map looks something like this:
realsende_depth_linear

The next step is just to embed the encoded image into the metadata of the original color image, and copy the final photo into the smartphone gallery. After that, you can just open the app, select the image, and refocus it!.

Posted in 3D, Image Processing, Photography, Programming.


Cross Platform Development for Intel Edison using CMake and Qt Creator(32 and 64 bits)

The Intel Edison is an amazing little device, but the current SDK only works out of the box with Eclipse, and creating new projects is not very easy.
In this post, I will show you how to use CMake and Qt Creator to develop for the Intel Edison remotely. This allows you to use the power of CMake to create new projects, and the convenience of Qt Creator, a great cross platform IDE for C/C++.
This guide should work for all major platforms (Windows, Mac, Linux, 32 and 64 bits) with little or no changes at all.

Here you can see a Windows machine running a debugging session with the Edison using Qt Creator:

debug

Required software

  • First of all, make sure that you have everything installed for your system to communicate with the Edison. The guys at SparkFun already covered this here.
  • Also, make sure that you have Qt Creator and CMake installed as well. Note that we are only using the Qt Creator IDE, so you don’t need to install the entire Qt SDK if you are not going to use it in your application.

Cross compiling

We are going to cross compile for the Edison. This means that we are going to build an executable (or library) for a target OS that is different from the host OS. In this case, the target OS is the Edison’s Yocto Linux and the host OS is the OS that you are running (Windows, Linux, Mac, 32 or 64 bits). Cross compiling means that the host OS compiles something for the target OS. Easy. In order to do this, you will need to have a set of applications for your host OS that allow you to compile for the target OS, as well as the libraries and include files of the target system. These things as a whole are called a toolchain.

The toolchain

Intel provides this toolchain for all the major OS on the main downloads site. Download the one for your OS under the section SDK Cross compile tools.

For Linux, just install it under the default directory:

/opt/poky-edison/1.6.1

For Windows, decompress the files (twice) and save them under C:/Edison.

You will end up with this directory structure for the 64 bits SDK:

C:/Edison/poky-edison-eglibc-x86_64-edison-image-core2-32-toolchain-1.6.1

And this directory structure for the 32 bits SDK:

C:/Edison/poky-edison-eglibc-i686-edison-image-core2-32-toolchain-1.6.1

Note that these locations are important because they will be referred to in the Qt Creator configuration and in the CMake script.

Make under Windows

There is a utility that was not included in the toolchain, a Make generator. The Eclipse version of the SDK uses an Eclipse internal tool to do the Make step. Since we are not using Eclipse, we will provide our own make generator tool. MinGW64 will help us here. It is a “Minimalist GNU for Windows”. Just download and install it in the default directory. Take a note of the installation directory because it is referred to in the CMake script, and it will probably change in future releases of MinGW64.

Qt Creator configuration

We are going to define a new device for development on Qt.
Open Qt Creator. Go to Tools->Options. On the left side, select Devices.

1_devices

Click on Add, and select Generic Linux Device. Click on Start Wizard.

2_StartWizrd
Set the name as Edison, or any name that you want here, the IP address of the Edison (something that may look like 192.168.1.42 for example), root as the username, and your current password. Click next when done.

3_newDeviceWizard

 

The device will then be tested for connectivity, and everything should be fine. If there is any problem, edit the settings and try again.

connectionTest

Now we are going to create a Qt kit to use with this device. Go to Build & Run and select the Compilers tab.

5_compilers

Once you are there, and select Add->GCC. Under Name, you can change it to anything you want. Under compiler path, you have to select the C++ compiler for your host OS. These are specified in the CMake script, but I will put them here as well for reference, but always make sure to double check your actual path:

Windows hosts(64 bits):

C:/Edison/poky-edison-eglibc-x86_64-edison-image-core2-32-toolchain-1.6.1/sysroots/x86_64-pokysdk-mingw32/usr/bin/i586-poky-linux/i586-poky-linux-g++.exe

Windows hosts(32 bits):

C:/Edison/poky-edison-eglibc-i686-edison-image-core2-32-toolchain-1.6.1/sysroots/i686-pokysdk-mingw32/usr/bin/i586-poky-linux/i586-poky-linux-g++.exe

Linux hosts (64 bits):

/opt/poky-edison/1.6.1/sysroots/x86_64-pokysdk-linux/usr/bin/i586-poky-linux/i586-poky-linux-g++

Linux hosts (32 bits):

/opt/poky-edison/1.6.1/sysroots/i686-pokysdk-linux/usr/bin/i586-poky-linux/i586-poky-linux-g++

 

6_compilerExecutable

Now go to the cmake tab and make sure that you have the cmake executable listed there.

8_cmake

Let’s add debugging capabilities. Go to the Debuggers tab and click on Add. Under Path, select the path to your gdb executable (This is part of the previously installed Mingw64 if you are on Windows, or just gdb if you are on Linux). You can change the name if you want.

debugConfig

Now we are ready to define a Qt Kit. Click on the Kits tab and click on Add.

Here you will set all the values that you created previously. Under Name you can put Edison or anything you want. Leave File system name blank. Under Device type select Generic Linux Device. This should automatically select Edison as your Device. Set your Sysroot to ({PATH_TO_SDK}/sysroots/core2-32-poky-linux), where PATH_TO_SDK depends on your OS as you saw earlier. Select GCC as your compiler (the one you previously defined). Under debugger set it to New Debugger, and finally under Qt version set it to None as well since we are not using the Qt SDK.

NewDebugger

Click OK and now Qt Creator is configured to build Edison projects.

The CMake script

This is what defines your project or application. This is going to be the script that describes what we want to do. In particular we will tell CMake that we want to do a cross compile for the Edison, and specify which libraries we want to use, as well as where our source files are, and what libraries we are going to use. There are two files, one that deals with all the details about cross compiling (Edison.cmake), and the other file that deals with your actual project (CMakeLists.txt).

You can download the project files here. I added a simple mraa example as the main.cpp that reads ADC A0 analog input, provided by Brendan Le Foll. Note that some boards like the mini breakout do not have this ability built in so they will return an error. That’s expected. The key here is to make the program compile and run on the Edison, not what the program actually does.

Edison.cmake

This file defines everything you need to deploy your application into the Edison. You will only need to setup this once, and you will be able to use this file in other projects. I will go through each section of the file explaining what it does.

The following code basically allows you to manually select which version of the SDK you are planning to use, 32 or 64 bits. Comment (add a # symbol) to the line you don’t want, and uncomment the line you want. Easy. By default it is set to use the 64 bits SDK.

#Are you using the 32 or the 64 bits version of the SDK?.
set(SDK_32bits FALSE) #Use this line for the 64 bits SDK
#set(SDK_32bits TRUE) #Use this line for the 32 bits SDK

This is straight forward. Just define where you want your application to be copied to into the Edison (currently it will be copied to /home/root).

#You can change this to any absolute folder in your Edison
#This is the folder where the executable will be copied to.
set(deploymentFolderAbsolutPath /home/root)

The following code defines the properties of the target OS. No need to change anything here.

#Set the target parameters
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_VERSION 3.10.17-poky-edison+)
set(CMAKE_SYSTEM_PROCESSOR i686)

This is where all the paths to the SDK are set. Your OS is identified by cmake, and the version of the SDK (32 or 64 bits) was manually defined earlier. You may only need to change the paths to your specific installations but it may just work as it is.

#Set the host parameters
if(WIN32)
    #Windows host
	if(${SDK_32bits})
		#32 bit SDK
		set(edison_sdk_root D:/Edison/poky-edison-eglibc-i686-edison-image-core2-32-toolchain-1.6.1)
		set(cross_compiler_sysroot ${edison_sdk_root}/sysroots/i686-pokysdk-mingw32)
		#MinGW make
		set(CMAKE_MAKE_PROGRAM "C:/Program Files/mingw-w64/i686-4.9.2-posix-dwarf-rt_v3-rev1/mingw32/bin/mingw32-make.exe")
	else()
		#64 bit SDK
		set(edison_sdk_root C:/Edison/poky-edison-eglibc-x86_64-edison-image-core2-32-toolchain-1.6.1)
		set(cross_compiler_sysroot ${edison_sdk_root}/sysroots/x86_64-pokysdk-mingw32)
		#MinGW make
		set(CMAKE_MAKE_PROGRAM "C:/Program Files (x86)/mingw-w64/i686-4.9.2-posix-dwarf-rt_v3-rev1/mingw32/bin/mingw32-make.exe")
	endif()
	set(CMAKE_C_COMPILER ${cross_compiler_sysroot}/usr/bin/i586-poky-linux/i586-poky-linux-gcc.exe)
	set(CMAKE_CXX_COMPILER ${cross_compiler_sysroot}/usr/bin/i586-poky-linux/i586-poky-linux-g++.exe)
else()
	#Linux host
	if(${SDK_32bits})
		#32 bit SDK
		set(edison_sdk_root /opt/poky-edison/1.6.1)
		set(cross_compiler_sysroot ${edison_sdk_root}/sysroots/i686-pokysdk-linux)
	else()
		#64 bit SDK
		set(edison_sdk_root /opt/poky-edison/1.6.1)
		set(cross_compiler_sysroot ${edison_sdk_root}/sysroots/x86_64-pokysdk-linux)
	endif()
	set(CMAKE_C_COMPILER ${cross_compiler_sysroot}/usr/bin/i586-poky-linux/i586-poky-linux-gcc)
	set(CMAKE_CXX_COMPILER ${cross_compiler_sysroot}/usr/bin/i586-poky-linux/i586-poky-linux-g++)
ENDIF(WIN32)

The following defines the rest of the cross platform configuration for the Edison, include folders, compiler flags, etc. No need to make changes here unless you know what you are doing.

set(CMAKE_SYSROOT ${edison_sdk_root}/sysroots/core2-32-poky-linux)
set(CMAKE_FIND_ROOT_PATH ${CMAKE_SYSROOT})
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

SET(CMAKE_C_FLAGS "-Os -g3 -Wall --sysroot=${CMAKE_SYSROOT} -m32 -march=i586 -ffunction-sections -fdata-sections"  CACHE STRING "" FORCE)
SET(CMAKE_CXX_FLAGS "-Os -g3 -Wall --sysroot=${CMAKE_SYSROOT} -m32 -march=i586 -ffunction-sections -fdata-sections"  CACHE STRING "" FORCE)

INCLUDE_DIRECTORIES(${CMAKE_SYSROOT}/usr/include)
INCLUDE_DIRECTORIES(${CMAKE_SYSROOT}/usr/include/c++)
INCLUDE_DIRECTORIES(${CMAKE_SYSROOT}/usr/include/c++/i586-poky-linux)

And finally this section defines a way to tell Qt to deploy a file

#This is for telling Qt to deploy files
file(WRITE "${CMAKE_SOURCE_DIR}/QtCreatorDeployment.txt" "/\n")
macro(add_deployment_file SRC DEST)
	file(RELATIVE_PATH path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
	file(APPEND "${CMAKE_SOURCE_DIR}/QtCreatorDeployment.txt" "${path}/${SRC}:${DEST}\n")
endmacro()
macro(add_deployment_directory SRC DEST)
	file(GLOB_RECURSE files RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${SRC}/*")
	foreach(filename ${files})
		get_filename_component(path ${filename} PATH)
		add_deployment_file("${filename}" "${DEST}/${path}")
	endforeach(filename)
endmacro()

CMakeLists.txt

This is the main file for your project. Every cmake project has at least one of these files.

First, I start defining the minimum version of cmake required for this project, and include the Edison.cmake file that I just described.

cmake_minimum_required(VERSION 2.6)

#This is where all the settings to compile for the Edison are stored.
INCLUDE(${CMAKE_SOURCE_DIR}/Edison.cmake)

This is the actual definition of the project. The name of the project (not the executable) is set as EdisonCrossCompile, and the project is set to compile either C or C++ sources. You can set exe_name to define the name for your application (currently it is set as executableName). I also included a script to update the SDK to the latest libraries versions. It is disabled by default as it overwrites the files. If you want to add this feature (at your own risk), uncomment the line. If you do that, every time CMake is called, you will update your SDK. This is usually at the beginning of the project, and when you click on Build->Run CMake, not when you build your project.

Also, any file that is put in this root directory with extension .cpp or .h will be included in the project. This is a simple way to start a new project. More complex projects can, of course, use the full cmake syntax to define their source structure. Also, I add libmraa as an example of how to use external libraries. The rest is to build and link the executable.

PROJECT(EdisonCrossCompile C CXX)

#This is for updating the local SDK to the latest version of mraa and upm
#Uncomment this line to update your SDK every time you run CMake (not every time you build your project)
#INCLUDE(${CMAKE_SOURCE_DIR}/updateSDK.cmake)

#This is the name of the compiled program. Change executableName to anything you want.
set(exe_name executableName)


#Here I am defining the source and header files of the project. You can just put .cpp and .h files into this folder and they will get compiled.
file(GLOB cpp_files *.cpp)
file(GLOB h_files *.h)

#Here you can include extra libraries for your application, like libmraa (http://iotdk.intel.com/docs/master/mraa/) for example
set(extra_libraries -lmraa)

#The actual compilation and linkage
add_executable(${exe_name} ${cpp_files} ${h_files})
target_link_libraries(${exe_name} ${extra_libraries})

Finally, this is used to copy the executable over to the Edison and execute it there.

#This tells Qt which file to copy over to the Edison.
file(RELATIVE_PATH relativeDir ${PROJECT_SOURCE_DIR} ${PROJECT_BINARY_DIR}/${exe_name})
add_deployment_file(${relativeDir} ${deploymentFolderAbsolutPath})

Opening your project in Qt Creator

Once you have your project configured, you will be able to open it under Qt Creator. Click on File->Open File or Project and select your CMakeLists.txt file.
You will see a windows asking you for the build location. This is where the build files are going to be created, and where by default your executable will be created. Usually this is a folder called build under the root of your project. Click Next.
7_buildPath
Now under Generator select Unix Generator(Edison). If you don’t see this option, go back to the section called Qt Creator configuration in this guide and do it properly.
Click on Run CMake and wait for a couple of seconds. Everything should work fine. Click on Finish when it is done.
8_runCmake
You should now see all your project files listed in the Qt Creator IDE.
9_ide

Click on the build icon (the brown hammer icon in the lower left section), and your project should build fine. You can actually see the compiled executable file for the Edison ready to be used in the build folder that you selected before.
10_executable

You can now manually copy this file over to the Edison and it will run fine there. But we will add an extra setting that will allow us to execute it directly from the Qt Creator IDE.
Click on Projects and click on Run (there is a small Build | Run selection right below the name of the project, on the upper side of the window). Under the Run section, under Run configuration, click on Add and select executableName(on Remote Device). Now you should be able to click on the build icon(Hammer symbol), or the build and run icon (green play symbol) and the application will get transferred to the Edison, executed remotely, and you will see the console output on the Qt Creator IDE.
11_externalRun

Troubleshooting

Here I will list the most common problems and their solutions.

CMake issues

If anything goes wrong, check that all the paths are correctly set in the cmake script. This is probably where most of the issues will be. You can use message(varName values is: ${varName}) to output the values of cmake variables for debugging.
Also, if you are stuck with an error while changing parameters it is a good idea to delete the entire build directory and start again as sometimes cached variables may still persist with the old values.

Qt Creator issues

If you see this error when building on Qt Creator on Windows: Could not start process “make”, it means that the mingw64 executable was not correctly recognized. The solution here is to go to Projects->Build, and under Build Steps click on Details. Under override command (Available in newer Qt Creator versions) click on Browse and select your mingw32-make.exe file. If you don’t have an override command available, just remove the make step and add a new custom process step and select your mingw32-make.exe file as the command. Repeat the same for the make clean step.

Posted in IoT, Programming, Qt.


Installing OpenCV 2.4.9 in Ubuntu 14.04 LTS

The latest Long Term Support version of Ubuntu(14.04 LTS) is out and a new version of OpenCV was recently released as well. This means that now is a great opportunity to update my OpenCV installation guide to the latest versions, Ubuntu 14.04 LTS and OpenCV 2.4.9.

viz

One of the great additions in this new release of OpenCV is the new viz module, which offers a nice 3D visualizer. This module is based on the great Visualization Toolkit library (VTK) and nicely prepares the users for the upcoming OpenCV 3.

In this guide, I will show you how to install OpenCV with a lot of the features it provides. Here are some of the things that are going to be enabled when you are finished following through with this installation tutorial:

  • viz module (3D visualization)
  • Qt version of the HighGUI module (Better 2D window interface with zoom, image saving capabilities, etc)
  • OpenGL support
  • C++ interface and examples
  • C interface and examples
  • Python interface and examples
  • Java interface and examples
  • Intel Threading Building Blocks (TBB)

Note: I have noticed some copies of my posts elsewhere, so make sure that you are reading this from the original source, at samontab dot com, accessible from here so that you don’t miss the comments.

OK, so the first step is to make sure that everything in the system is updated and upgraded. Open the terminal and write this:

sudo apt-get update
sudo apt-get upgrade

Now, you need to install many dependencies, such as support for reading and writing image files, drawing on the screen, some needed tools, other libraries, etc… This step is very easy, you only need to write the following command in the Terminal:

sudo apt-get install build-essential libgtk2.0-dev libjpeg-dev libtiff4-dev libjasper-dev libopenexr-dev cmake python-dev python-numpy python-tk libtbb-dev libeigen3-dev yasm libfaac-dev libopencore-amrnb-dev libopencore-amrwb-dev libtheora-dev libvorbis-dev libxvidcore-dev libx264-dev libqt4-dev libqt4-opengl-dev sphinx-common texlive-latex-extra libv4l-dev libdc1394-22-dev libavcodec-dev libavformat-dev libswscale-dev default-jdk ant libvtk5-qt4-dev

Time to get the OpenCV 2.4.9 source code:

cd ~
wget http://sourceforge.net/projects/opencvlibrary/files/opencv-unix/2.4.9/opencv-2.4.9.zip
unzip opencv-2.4.9.zip
cd opencv-2.4.9

Now we have to generate the Makefile by using cmake. In here we can define which parts of OpenCV we want to compile. Since we want to use the viz module, Python, Java, TBB, OpenGL, Qt, work with videos, etc, here is where we need to set that. Just execute the following line at the terminal to create the appropriate Makefile. Note that there are two dots at the end of the line, it is an argument for the cmake program and it means the parent directory (because we are inside the build directory, and we want to refer to the OpenCV directory, which is its parent).

mkdir build
cd build
cmake -D WITH_TBB=ON -D BUILD_NEW_PYTHON_SUPPORT=ON -D WITH_V4L=ON -D INSTALL_C_EXAMPLES=ON -D INSTALL_PYTHON_EXAMPLES=ON -D BUILD_EXAMPLES=ON -D WITH_QT=ON -D WITH_OPENGL=ON -D WITH_VTK=ON ..

OpenCVConfig

Check that the above command produces no error and that in particular it reports FFMPEG as YES. If this is not the case you will not be able to read or write videos. Check that Java, Python, TBB, OpenGL, V4L, OpenGL and Qt are all detected correctly.

OpenCVmodulesToBeBuilt

Make sure you scroll up and check that the modules that are going to be built are these:
core flann imgproc highgui features2d calib3d ml video legacy objdetect photo gpu ocl nonfree contrib java python stitching superres ts videostab viz.

If anything is wrong, go back, correct the errors by maybe installing extra packages and then run cmake again.

Now, you are ready to compile and install OpenCV 2.4.9:

make
sudo make install

Now you have to configure OpenCV. First, open the opencv.conf file with the following code:

sudo gedit /etc/ld.so.conf.d/opencv.conf

Add the following line at the end of the file(it may be an empty file, that is ok) and then save it:

/usr/local/lib

Run the following code to configure the library:

sudo ldconfig

Now you have to open another file:

sudo gedit /etc/bash.bashrc

Add these two lines at the end of the file and save it:

PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig
export PKG_CONFIG_PATH

Finally, close the console and open a new one, restart the computer or logout and then login again. OpenCV will not work correctly until you do this.

Now you have OpenCV 2.4.9 installed in your computer with 3D visualization, Python, Java, TBB, OpenGL, video, and Qt support.

Check out the cool Qt interface which provides image viewing capabilities with zoom, as well as the ability to save the current image with just one click.

If you zoom in enough, you can see the RGB (or intensity) values for each pixel.

Now let’s build some samples included in OpenCV:

cd ~/opencv-2.4.9/samples/c
chmod +x build_all.sh
./build_all.sh
These examples use the old C interface:
./facedetect --cascade="/usr/local/share/OpenCV/haarcascades/haarcascade_frontalface_alt.xml" --scale=1.5 lena.jpg

./facedetect --cascade="/usr/local/share/OpenCV/haarcascades/haarcascade_frontalface_alt.xml" --nested-cascade="/usr/local/share/OpenCV/haarcascades/haarcascade_eye.xml" --scale=1.5 lena.jpg

The following examples use the new C++ interface:

~/opencv-2.4.9/build/bin/cpp-example-grabcut ~/opencv-2.4.9/samples/cpp/lena.jpg

~/opencv-2.4.9/build/bin/cpp-example-calibration_artificial

Now let’s run some Python code:

python ~/opencv-2.4.9/samples/python2/turing.py

Now you can build a Java sample using ant for example. (Make sure that you change /home/samontab/ with your actual home directory):

cd ~/opencv-2.4.9/samples/java/ant
ant -DocvJarDir=/home/samontab/opencv-2.4.9/build/bin -DocvLibDir=/home/samontab/opencv-2.4.9/build/lib

java_ant

Now let’s read a video and use OpenGL with Qt through this great sample that detects the features from the video, then estimates the 3D location of the structure using POSIT, and finally uses OpenGL to draw in 3D (great sample Javier):

cd ~/opencv-2.4.9/samples/cpp/Qt_sample
mkdir build
cd build
cmake ..
make
./OpenGL_Qt_Binding

posit

And finally, let’s build a sample using the 3D visualization module viz:

cd ~/opencv-2.4.9/samples/cpp/tutorial_code/viz
g++ -o widget_pose `pkg-config opencv --cflags` widget_pose.cpp `pkg-config opencv --libs`
./widget_pose

viz

As you can see, now you can use OpenCV with C++, C, Python, and Java. The Qt enhanced 2D interface is enabled, 3D data can be displayed using OpenGL directly, or using the new viz module. Multi threading functionality is enabled using TBB. Also, video support is enabled as well.

If you correctly finished this tutorial, you are now ready to do some nice computer vision work.

Posted in Image Processing, Open Source, Programming, Qt.


My New OpenCV Computer Vision Application Programming Video Course

OpenCV_Computer_Vision_Application_ProgrammingAfter writing articles helping people to install OpenCV here on this blog, some asked me to share a little more about OpenCV, not just the installation part. I started thinking about a way to deliver this information when suddenly Packt publishing contacted me about making a video course tutorial for OpenCV. I knew it was the perfect solution, so I immediately said yes. After the publication of the video course at Packt, I was glad to see it appear also in the official website of OpenCV.

This video course starts right at the beginning, with an explanation of what OpenCV is and its installation. Then, you learn all the basics of OpenCV, followed by some of the basics of image processing. Then, you will be ready to detect objects, recognize faces, train your own detector, remove distortion from your photos, and play with images in 3D.

Here are the things that you will learn with this video course:

  • Learn what OpenCV is and how to install it on Linux and Windows
  • Development with OpenCV using the recommended C++ interface, as well as an introduction to the Python and C interfaces.
  • Segment objects in your images using grabcut and watershed, cluster your data, and spot interesting points in the image
  • Merge different images into a single panorama using the technique called panorama stitching
  • Detect common objects in your images, like faces, eyes, or people
  • Train your own object detector to detect custom objects
  • Recognize a face among many others
  • Learn to calibrate your camera
  • Remove or reduce the distortion of an image caused by the lens, commonly known as barrel distortion
  • Change the perspective of an image to match a different 3D pose
  • Create an image that represents depth information of the scene using stereoscopic images

This is an easy to follow video course for people that are just starting with OpenCV. Even people that are used to OpenCV may benefit from some of the more advanced videos since you can just pick and choose which video you want to learn from.

You can check out the video course here.

Posted in 3D, Image Processing, Open Source, Programming.


How to make HD screencasts using free tools

A screencast is basically a video of your computer screen with your voice as the audio. A screencast is very useful for tutorials or teaching how to use something with your computer.
There are a couple of paid programs that allow you to create screencasts, but in this post I will show you how to make them using only free tools(although you need to be running Windows for this tutorial, which is not free, sorry about that).

The first step is to download and install the codec that will allow us to compress the video. We are going to use one of the best standards for HD video: H.264/MPEG-4 AVC. The specific implementation to install here is x264vfw.

You need to install the 32 bits version (even if you have a 64 bits system). After you install the 32 bits version, try to install the 64 bits version as well (you may not be able to do it in a 32 bit system).

Now, install the latest version of camstudio. It is a great program that will allow you to record your voice and your desktop. Make sure that you also install the Lossless Video Codec as well.

OK, now you should have camstudio and its Lossless Video Codec installed. Make sure that you have your mic and/or video camera connected to your computer. Now we are going to configure camstudio to make it screencast ready.

Start camstudio and set the following options:

  • Options-> Record audio from microphone
  • Options-> Program Options -> Play AVI file when recording stops -> Do not play AVI file
  • Options-> Program Options -> Name of AVI file -> Automatic file naming (use date and time)
  • Options-> Program Options -> Minimize program on start recording (not checked)
  • Options-> Program Options -> Directory for recording -> Use user specified directory… (and specify a directory where you want your videos to be saved)

Now, go to Options-> Video options, and select the following:
camstudio_lossless

  • Compressor: CamStudio Lossless Codec 1.5 (or newer)
  • Quality 100
  • Select Auto Adjust
  • Move the slider to where you have a Playback rate of 25 frames / second (the default one is 200 fps!)

Now go to Options-> Audio options-> Audio options for microphone, and select the following:
camstudio_audio

  • Audio Capture Device: Select your device
  • Recording Format, select 44.1kHz, stereo, 16 bit.
  • Compressed format: PCM

Now we are going to set which area of the screen to record. Go to Region->Fixed Region and set these values:
camstudio_fixedregion

  • Left: 80
  • Top: 25
  • Width: 1280
  • Height: 720

The most important thing to note here is the width and height of 1280 and 720 respectively. This is widescreen HD format (720p) and it is very important that you use these values. If you use other, non standard values, you will end up with problems in your video. If you have a larger screen, you may want to use a width of 1920 and a height of 1080, which is even better (1080p). The other values, left and top, should be set depending on the actual area that you want to record of your desktop. Remember that left plus width and top plus height should be less or equal to your total screen width and height respectively.

Alternatively to selecting a specific region in your screen, you can first adjust your screen resolution to 1280×720 (or 1920×1080), and select Region-> Full Screen instead. I prefer using a region because you can show your software in your native resolution, and also you have the option to not show the clock, and the menu bar, etc. This is specially important for continuity when editing videos later.

OK, now go ahead and make a couple of quick tests. Click on the record button(red circle), start talking, moving your mouse, opening folders, etc. After some seconds, press stop(blue square). After a moment, a video should be waiting for you in the location you specified earlier without you having to do anything else. Go there and check it. Make sure that the video came out OK, that you can see the areas that you want, that your voice is loud and clear, that the video resolution and frame rate are what you expected, and that you are comfortable with this recording procedure. Also, always make sure that you have at least a couple of seconds at the beginning and at the end of your recordings. Specially at the end, because the encoding technique that we will use cuts one or two seconds at the end, depending on the length of your video.

Before you start doing real screencasts, let’s check a couple of issues. Camstudio is a great program, but sometimes it crashes, and it forgets all your settings, which is unfortunate (incredibly annoying and frustrating). That’s why it is good to save your preferences, which are stored in four files located at the installation directory (C:\Program Files (x86)\CamStudio 2.6b). Once you have set all your preferences, just copy these four files: Camdata.ini, CamLayout.ini, CamShapes.ini, CamStudio.ini into a safe place (C:\Users\Public\Documents\camstudioconfig for example). If camstudio crashes and forgets your preferences, simply close the software, copy your files back and continue normally.

After you record a couple of videos, you can edit them. We are going to use a video editor called virtualdub. Download and open it. Drag and drop a video there. You can now see all the frames that compose the video. You can select some frames and delete them for example. Also, you can append another video. You can also do many other things, but appending videos and removing frames is all you need to create a nice screencast. It is important that all your videos are recorded with the same settings, like width, height, even sound should be the same.

Before you edit a video, always check these settings in virtualdub:

  • Make sure that Video->Full processing mode is selected
  • Make sure that Audio->Direct stream copy is selected

Now, go to Video-> compression and select x264vfw:
virtualdub_compression

Click on configure and set the following options:
virtualdub_configuration

  • enable zero latency
  • enable virtualdub hack

Click OK, OK
Now you can edit the screencast by appending your videos, and removing frames. You can take a look at what virtualdub has to offer here.
Once you are finished editing your screencast, you can go to File->save as avi. It will generate the first version of your screencast, but we need to compress it even further and change it into a different format.

Now comes the final step. Install and open vlc and select Media->Convert/Save. Under file selection, click Add and select the AVI file you just created with virtualdub. Click on Convert/Save. On Destination folder, copy the same file path as the source, but end the filename with mp4 instead of avi. Under profile, make sure that Video H.264 + AAC (MP4) is selected. Click on start, and after some minutes, you will have your HD screencast ready in the same folder that you selected at the beginning.

Note:
If you need to make a screencast of Linux, or other operating system, you can use virtualbox. Just use it as a normal windows application, and everything will be recorded. Make sure that your region is set so that it covers the area that you want to record. If you are interested in a native application, you can take a look at recordmydesktop.

Note 2:
If you already have a recording in camstudio and want to change the frame rate, you can do that easily with virtualdub. Open the video in virtualdub, go to video, select direct stream copy. Go to video, select frame rate.
Under frame rate conversion, select convert to fps, and put 25 (or the desired value). Go to file, select save as avi. It should process the file quickly.

Posted in General, Open Source.


Technical reviewer of a book about computer vision projects solved using OpenCV

There is a new book about OpenCV:

Mastering OpenCV with Practical Computer Vision Projects.

7829OS_Mastering OpenCV with Practical Computer Vision Projects.jpgI was honored to be invited to participate as a technical reviewer of the book, and it was really fun to read. Each chapter shows a specific real world problem and teaches you how it can be solved using OpenCV. This is the work of seven authors, and the projects are great. I really recommend this book if you are already using OpenCV, because it deals with more advanced details, and in some cases it shows you how to use other libraries to fill in where OpenCV lacks for specific purposes.

Some of the cool things that this book covers are how to make OpenCV powered apps for Android and iOS, interfacing OpenCV with the Kinect, number plate detection, optical character recognition (OCR), augmented reality, structure from motion, face detection and tracking.

If you are interested in using OpenCV in real world scenarios, this book will definitely help you. Although you will need to know the basics of image processing or computer vision and programming first as this book is mostly focused on the actual practical projects.

Posted in 3D, Image Processing, Open Source, Programming.


Playing with astrophotography

One of the easiest subjects to photograph in the night sky is the Moon. Everyone knows where it is, and it looks large and bright from the earth(well, unless it is in the new moon phase).
moon

Yesterday, Jupiter was passing close to the Moon, as seen from the Earth.
jupiter_close_to_the_moon

I tried to get a close up of Jupiter and the Moon in the same frame, but the brightness of the Moon was overexposing the photograph.
jupiter_moon_too_bright

When I properly exposed the Moon, Jupiter was not very visible.
jupiter_moon_too_dark

I could have used HDR techniques here to merge differently exposed photos into a single one that had Jupiter and the Moon properly exposed, but I changed my mind and started focusing on Jupiter alone.

Removing the Moon from the frame allowed me to gather more light from Jupiter and its surroundings, and that’s when four other objects appeared in my picture.
jupiter_four_moons

Checking with Stellarium, I confirmed that these objects were Europa, Io, Ganymede, and Callisto. They are the four largest moons of Jupiter!.

I never thought that I would be able to photograph the moons of Jupiter from the Earth using my own camera, but there they are!.
moons_of_jupiter

Here is the previous photo aligned with the simulated sky from Stellarium. Cool!
jupiter_moons_aligned

Posted in General, Photography.


Using OpenCV in Sage

OpenCV is an awesome free open source library for computer vision, image processing, and even some machine learning.
Sage is an awesome free open source mathematical software system that integrates many other libraries and software into a single, easy to use interface for all your math needs.

I really like both projects, and have been using them separately, but in this post I will show you how you can use OpenCV in the Sage environment. This means that you will be able to use all the power of the Sage Notebook Server, NumPy, SciPy, Matplotlib, PIL (Python Imaging Library), R, and almost a hundred other open source packages for all your image processing needs.

sage0

The first step is to install the latest version of OpenCV in your system with the new Python bindings. You must have the Python bindings working correctly for this to work. The easiest way of doing this is to follow my own tutorial for installing OpenCV. You can easily change the version for installing the latest OpenCV version that is out there, the tutorial should still work. I used it to install OpenCV 2.4.3 in Ubuntu 12.10, and it worked perfectly.

To make sure that OpenCV is installed correctly, go to the console and start Python by running this command:

python

You should see the python console waiting for you. There, write the following:

import cv2

Importing OpenCV cv2 module into python
If no error appeared, everything is great, you can continue. If there are some errors, you need to fix them first. You can exit with Control-D, or by typing exit().

By the way, currently there are two OpenCV Python modules, cv and cv2. The legacy C version API of OpenCV can be accessed via cv, and the new C++ interface of OpenCV is available via cv2. Therefore, you are encouraged to use cv2 because that’s where the good stuff is.

Now that we have OpenCV with Python correctly installed, we need to install Sage.
First, check if you are running a 32 or 64 bit system by using this command:

uname -m

You will see x86_64, which means 64 bits or something like i686 or i386, which means 32 bits.

Now we need to download Sage from here. You can pick where to download from, as well as if you want the 32 or 64 bits version. I usually download it from the University of Washington which is where the lead developer of Sage, William Stein, teaches math but you can choose any other location that you like. You are looking for a file that has a name similar to this one: sage-5.4.1-linux-32bit-ubuntu_12.04.1_lts-i686-Linux.tar.lzma. For the next steps, I will be using that file. Remember that you may need to change the file or folder names for your specific situation.

After you download the file, you will need to extract it. On the console, change your directory to where you downloaded the file and extract it:

cd ~/Downloads
tar --lzma -xf sage-*

Now you will have to wait for a few minutes for it to decompress a lot of files. Notice how I did not use the verbose option for extracting (I used -xf instead of -xvf). This is because it takes a lot longer to decompress when it has to output every file name. Since nothing is being printed on the screen, it is faster but you may think that nothing is happening. Don’t worry, everything is fine. Just leave it alone until it finishes decompressing.

Now we are going to configure it so that we can call it from anywhere.

mv sage-5.4.1-linux-32bit-ubuntu_12.04.1_lts-i686-Linux sage-5.4.1
sudo mv sage-5.4.1 /opt/
sudo cp /opt/sage-5.4.1/sage /usr/local/bin/
sudo gedit /usr/local/bin/sage
  1. Copy the line that says #SAGE_ROOT=/path/to/sage-version.
  2. Remove the comment character (#) in the line you just copied.
  3. Change the path to /opt/sage-5.4.1 (or the version you are installing).
  4. Save the file and close the editor.

sage_root

And now you can start Sage by using this command anywhere:

sage

You will have to wait for a few moments this first time you run it. You should then see the command prompt of Sage waiting for your input. Exit for now by running this command:

exit

If everything is fine, you can now delete the downloaded file to save some space:

rm ~/Downloads/sage-5.4.1-linux-32bit-ubuntu_12.04.1_lts-i686-Linux.tar.lzma

OK, so now we also have Sage correctly installed in the system.

Let’s install OpenCV into Sage:

cd /usr/local/lib/python2.7/dist-packages/
cp cv2.so /opt/sage-5.4.1/local/lib/python2.7/site-packages/
cp cv.py /opt/sage-5.4.1/local/lib/python2.7/site-packages/
cd ~

To make sure that OpenCV is correctly installed in Sage, go to the console and start Sage by running this command:

sage

You should see the Sage console waiting for you. There, write the following:

import cv2

If no error appeared, everything is great, you can continue. If there are some errors, you need to fix them first. At this point, it should report the following error:

RuntimeError: module compiled against API version 6 but this version of numpy is 4.

This happens because Sage(as of v5.4.1) still comes with an older version of NumPy(v1.5.1). You can tell which is the included version of NumPy by doing this in Sage:

import numpy
numpy.version.version
exit

So, if you have a newer version of Sage (this fix is scheduled for v5.6), and did not get the error, you can skip the next step.

Let’s update Sage’s NumPy to v1.7.0b2 so that we can use OpenCV with it:

cd ~
wget https://spkg-upload.googlecode.com/files/numpy-1.7.0b2.spkg
sage -f numpy-1.7.0b2.spkg

Now you should be able to run OpenCV functions in Sage.

Let’s use the Sage Notebook interface with OpenCV.
Start sage, run notebook(), and follow the instructions for using the notebook interface. For more advanced options for the notebook server, you can check here.

Once you have a new worksheet open, define some functions to convert arrays between NumPy, OpenCV and PIL:

import cv2, numpy, Image, ImageDraw

def cv2np(image):    
    rgbimage = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    return rgbimage/255.
    
def pil2np(image):
    rgbimage = numpy.array(image)
    return rgbimage/255.
    
def cv2pil(image):
    rgbimage = cv2.cvtColor(lenna, cv2.COLOR_BGR2RGB)
    pilimage = Image.fromarray(rgbimage)
    return pilimage

Now, in another cell let’s load an image and find a face using the trained frontal face model. (Make sure that you change the path to your image):

image_path = "/home/username/OpenCV-2.4.3/samples/c/lena.jpg"
lenna = cv2.imread(image_path, cv2.CV_LOAD_IMAGE_COLOR)
cascade_fn = "/usr/local/share/OpenCV/haarcascades/haarcascade_frontalface_alt.xml"
cascade = cv2.CascadeClassifier(cascade_fn)
gray = cv2.cvtColor(lenna, cv2.COLOR_BGR2GRAY)
rects = cascade.detectMultiScale(gray, scaleFactor=1.2, minNeighbors=4, minSize=(30,30))
for x1, y1, width, height in rects:    
    faces_rectangle = [x1, y1, x1 + width, y1 + height]
faces_rectangle

If you use lena.jpg, you should see the following result:

[217, 203, 386, 372]

Which is the location of the bounding box of the detected face of Lenna.
Now, let’s use PIL to draw the detection:

lenna_pil = cv2pil(lenna)
draw = ImageDraw.Draw(lenna_pil)
draw.ellipse((x1, y1, x1+width, y1+height), outline="green")
draw.ellipse((x1+1, y1+1, x1+width-1, y1+height-1), outline="green")
draw.ellipse((x1+2, y1+2, x1+width-2, y1+height-2), outline="green")

To be able of changing the thickness of the ellipse in PIL, you need to install the aggdraw module, so I just drew three ellipses instead.

Finally, it is time to show the detection:

matrix_plot(pil2np(lenna_pil)).show( axes=True, frame=True, figsize=8, aspect_ratio=1)

sage_opencv
And that’s it. This is just a simple example of object detection in OpenCV using the Sage environment. Feel free to explore Sage and OpenCV now that you can use them in the same interface.

Posted in Image Processing, Open Source.