Visualization of the NARF descriptor and descriptor distances

This tutorial is about the visualization of how the NARF descriptor is calculated and to test how the descriptor distances between certain points in a range image behave. Compared to the other tuturials, this one is not really about the code, but about trying the program and looking at the visualization. Of course, nothing keeps you from having a look at it anyway.

The code

First, create a file called, let’s say, narf_descriptor_visualization.cpp in your favorite editor, and place the following code inside it:

  1/* \author Bastian Steder */
  2
  3#include <iostream>
  4
  5#include <pcl/point_cloud.h>
  6#include <pcl/io/pcd_io.h>
  7#include <pcl/visualization/range_image_visualizer.h>
  8#include <pcl/range_image/range_image.h>
  9#include <pcl/features/narf.h>
 10#include <pcl/console/parse.h>
 11
 12float angular_resolution = 0.5f;
 13int rotation_invariant = 0;
 14float support_size = 0.3f;
 15int descriptor_size = 36;
 16pcl::RangeImage::CoordinateFrame coordinate_frame = pcl::RangeImage::CAMERA_FRAME;
 17bool setUnseenToMaxRange = false;
 18
 19typedef pcl::PointXYZ PointType;
 20
 21void 
 22printUsage (const char* progName)
 23{
 24  std::cout << "\n\nUsage: "<<progName<<" [options] <scene.pcd>\n\n"
 25            << "Options:\n"
 26            << "-------------------------------------------\n"
 27            << "-r <float>   angular resolution in degrees (default "<<angular_resolution<<")\n"
 28            << "-s <float>   support size for the interest points (diameter of the used sphere - "
 29            <<                                                     "default "<<support_size<<")\n"
 30            << "-d <int>     descriptor size (default "<<descriptor_size<<")\n"
 31            << "-c <int>     coordinate frame of the input point cloud (default "<< (int)coordinate_frame<<")\n"
 32            << "-o <0/1>     switch rotational invariant version of the feature on/off"
 33            <<               " (default "<< (int)rotation_invariant<<")\n"
 34            << "-m           set unseen pixels to max range\n"
 35            << "-h           this help\n"
 36            << "\n\n";
 37}
 38
 39int 
 40main (int argc, char** argv)
 41{
 42  // --------------------------------------
 43  // -----Parse Command Line Arguments-----
 44  // --------------------------------------
 45  if (pcl::console::find_argument (argc, argv, "-h") >= 0)
 46  {
 47    printUsage (argv[0]);
 48    return 0;
 49  }
 50  if (pcl::console::find_argument (argc, argv, "-m") >= 0)
 51  {
 52    setUnseenToMaxRange = true;
 53    std::cout << "Setting unseen values in range image to maximum range readings.\n";
 54  }
 55  if (pcl::console::parse (argc, argv, "-o", rotation_invariant) >= 0)
 56    std::cout << "Switching rotation invariant feature version "<< (rotation_invariant ? "on" : "off")<<".\n";
 57  int tmp_coordinate_frame;
 58  if (pcl::console::parse (argc, argv, "-c", tmp_coordinate_frame) >= 0)
 59  {
 60    coordinate_frame = pcl::RangeImage::CoordinateFrame (tmp_coordinate_frame);
 61    std::cout << "Using coordinate frame "<< (int)coordinate_frame<<".\n";
 62  }
 63  if (pcl::console::parse (argc, argv, "-s", support_size) >= 0)
 64    std::cout << "Setting support size to "<<support_size<<".\n";
 65  if (pcl::console::parse (argc, argv, "-d", descriptor_size) >= 0)
 66    std::cout << "Setting descriptor size to "<<descriptor_size<<".\n";
 67  if (pcl::console::parse (argc, argv, "-r", angular_resolution) >= 0)
 68    std::cout << "Setting angular resolution to "<<angular_resolution<<"deg.\n";
 69  angular_resolution = pcl::deg2rad (angular_resolution);
 70  
 71
 72  // -----------------------
 73  // -----Read pcd file-----
 74  // -----------------------
 75  pcl::PointCloud<PointType>::Ptr point_cloud_ptr (new pcl::PointCloud<PointType>);
 76  pcl::PointCloud<PointType>& point_cloud = *point_cloud_ptr;
 77  pcl::PointCloud<pcl::PointWithViewpoint> far_ranges;
 78  Eigen::Affine3f scene_sensor_pose (Eigen::Affine3f::Identity ());
 79  std::vector<int> pcd_filename_indices = pcl::console::parse_file_extension_argument (argc, argv, "pcd");
 80  if (!pcd_filename_indices.empty ())
 81  {
 82    std::string filename = argv[pcd_filename_indices[0]];
 83    if (pcl::io::loadPCDFile (filename, point_cloud) == -1)
 84    {
 85      std::cout << "Was not able to open file \""<<filename<<"\".\n";
 86      printUsage (argv[0]);
 87      return 0;
 88    }
 89    scene_sensor_pose = Eigen::Affine3f (Eigen::Translation3f (point_cloud.sensor_origin_[0],
 90                                                               point_cloud.sensor_origin_[1],
 91                                                               point_cloud.sensor_origin_[2])) *
 92                        Eigen::Affine3f (point_cloud.sensor_orientation_);
 93    std::string far_ranges_filename = pcl::getFilenameWithoutExtension (filename)+"_far_ranges.pcd";
 94    if (pcl::io::loadPCDFile (far_ranges_filename.c_str (), far_ranges) == -1)
 95      std::cout << "Far ranges file \""<<far_ranges_filename<<"\" does not exists.\n";
 96  }
 97  else
 98  {
 99    std::cout << "\nNo *.pcd file for scene given.\n\n";
100    printUsage (argv[0]);
101    return 1;
102  }
103  
104  // -----------------------------------------------
105  // -----Create RangeImage from the PointCloud-----
106  // -----------------------------------------------
107  float noise_level = 0.0;
108  float min_range = 0.0f;
109  int border_size = 1;
110  pcl::RangeImage::Ptr range_image_ptr (new pcl::RangeImage);
111  pcl::RangeImage& range_image = *range_image_ptr;
112  range_image.createFromPointCloud (point_cloud, angular_resolution, pcl::deg2rad (360.0f), pcl::deg2rad (180.0f),
113                                   scene_sensor_pose, coordinate_frame, noise_level, min_range, border_size);
114  range_image.integrateFarRanges (far_ranges);
115  if (setUnseenToMaxRange)
116    range_image.setUnseenToMaxRange ();
117  
118  // Extract NARF features:
119  std::cout << "Now extracting NARFs in every image point.\n";
120  std::vector<std::vector<pcl::Narf*> > narfs;
121  narfs.resize (range_image.points.size ());
122  int last_percentage=-1;
123  for (unsigned int y=0; y<range_image.height; ++y)
124  {
125    for (unsigned int x=0; x<range_image.width; ++x)
126    {
127      int index = y*range_image.width+x;
128      int percentage = (int) ((100*index) / range_image.points.size ());
129      if (percentage > last_percentage)
130      {
131        std::cout << percentage<<"% "<<std::flush;
132        last_percentage = percentage;
133      }
134      pcl::Narf::extractFromRangeImageAndAddToList (range_image, x, y, descriptor_size,
135                                                    support_size, rotation_invariant != 0, narfs[index]);
136      //std::cout << "Extracted "<<narfs[index].size ()<<" features for pixel "<<x<<","<<y<<".\n";
137    }
138  }
139  std::cout << "100%\n";
140  std::cout << "Done.\n\n Now you can click on points in the image to visualize how the descriptor is "
141       << "extracted and see the descriptor distances to every other point..\n";
142  
143  //---------------------
144  // -----Show image-----
145  // --------------------
146  pcl::visualization::RangeImageVisualizer range_image_widget ("Scene range image"),
147                                           surface_patch_widget("Descriptor's surface patch"),
148                                           descriptor_widget("Descriptor"),
149                                           descriptor_distances_widget("descriptor distances");
150  range_image_widget.showRangeImage (range_image);
151  //range_image_widget.visualize_selected_point = true;
152
153  //--------------------
154  // -----Main loop-----
155  //--------------------
156  while (true) 
157  {
158    range_image_widget.spinOnce ();  // process GUI events
159    surface_patch_widget.spinOnce ();  // process GUI events
160    descriptor_widget.spinOnce ();  // process GUI events
161    pcl_sleep(0.01);
162    
163    //if (!range_image_widget.mouse_click_happened)
164      continue;
165    //range_image_widget.mouse_click_happened = false;
166    //float clicked_pixel_x_f = range_image_widget.last_clicked_point_x,
167          //clicked_pixel_y_f = range_image_widget.last_clicked_point_y;
168    int clicked_pixel_x, clicked_pixel_y;
169    //range_image.real2DToInt2D (clicked_pixel_x_f, clicked_pixel_y_f, clicked_pixel_x, clicked_pixel_y);
170    if (!range_image.isValid (clicked_pixel_x, clicked_pixel_y))
171      continue;
172      //Vector3f clicked_3d_point;
173      //range_image.getPoint (clicked_pixel_x, clicked_pixel_y, clicked_3d_point);
174    
175    //surface_patch_widget.show (false);
176    //descriptor_widget.show (false);"
177    
178    int selected_index = clicked_pixel_y*range_image.width + clicked_pixel_x;
179    pcl::Narf narf;
180    if (!narf.extractFromRangeImage (range_image, clicked_pixel_x, clicked_pixel_y,
181                                                                         descriptor_size, support_size))
182    {
183      continue;
184    }
185    
186    int surface_patch_pixel_size = narf.getSurfacePatchPixelSize ();
187    float surface_patch_world_size = narf.getSurfacePatchWorldSize ();
188    surface_patch_widget.showFloatImage (narf.getSurfacePatch (), surface_patch_pixel_size, surface_patch_pixel_size,
189                                         -0.5f*surface_patch_world_size, 0.5f*surface_patch_world_size, true);
190    float surface_patch_rotation = narf.getSurfacePatchRotation ();
191    float patch_middle = 0.5f* (float (surface_patch_pixel_size-1));
192    float angle_step_size = pcl::deg2rad (360.0f)/narf.getDescriptorSize ();
193    float cell_size = surface_patch_world_size/float (surface_patch_pixel_size),
194          cell_factor = 1.0f/cell_size,
195          max_dist = 0.5f*surface_patch_world_size,
196          line_length = cell_factor* (max_dist-0.5f*cell_size);
197    for (int descriptor_value_idx=0; descriptor_value_idx<narf.getDescriptorSize (); ++descriptor_value_idx)
198    {
199      float angle = descriptor_value_idx*angle_step_size + surface_patch_rotation;
200      //surface_patch_widget.markLine (patch_middle, patch_middle, patch_middle+line_length*sinf (angle),
201                                     //patch_middle+line_length*-std::cos (angle), pcl::visualization::Vector3ub (0,255,0));
202    }
203    std::vector<float> rotations, strengths;
204    narf.getRotations (rotations, strengths);
205    float radius = 0.5f*surface_patch_pixel_size;
206    for (unsigned int i=0; i<rotations.size (); ++i)
207    {
208      //surface_patch_widget.markLine (radius-0.5, radius-0.5, radius-0.5f + 2.0f*radius*sinf (rotations[i]),
209                                                //radius-0.5f - 2.0f*radius*std::cos (rotations[i]), pcl::visualization::Vector3ub (255,0,0));
210    }
211    
212    descriptor_widget.showFloatImage (narf.getDescriptor (), narf.getDescriptorSize (), 1, -0.1f, 0.3f, true);
213
214    //===================================
215    //=====Compare with all features=====
216    //===================================
217    const std::vector<pcl::Narf*>& narfs_of_selected_point = narfs[selected_index];
218    if (narfs_of_selected_point.empty ())
219      continue;
220    
221    //descriptor_distances_widget.show (false);
222    float* descriptor_distance_image = new float[range_image.points.size ()];
223    for (unsigned int point_index=0; point_index<range_image.points.size (); ++point_index)
224    {
225      float& descriptor_distance = descriptor_distance_image[point_index];
226      descriptor_distance = std::numeric_limits<float>::infinity ();
227      std::vector<pcl::Narf*>& narfs_of_current_point = narfs[point_index];
228      if (narfs_of_current_point.empty ())
229        continue;
230      for (unsigned int i=0; i<narfs_of_selected_point.size (); ++i)
231      {
232        for (unsigned int j=0; j<narfs_of_current_point.size (); ++j)
233        {
234          descriptor_distance = (std::min)(descriptor_distance,
235                                           narfs_of_selected_point[i]->getDescriptorDistance (*narfs_of_current_point[j]));
236        }
237      }
238    }
239    descriptor_distances_widget.showFloatImage (descriptor_distance_image, range_image.width, range_image.height,
240                                               -std::numeric_limits<float>::infinity (), std::numeric_limits<float>::infinity (), true);
241    delete[] descriptor_distance_image;
242  }
243}

Compiling and running the program

Add the following lines to your CMakeLists.txt file:

 1cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
 2
 3project(narf_descriptor_visualization)
 4
 5find_package(PCL 1.3 REQUIRED)
 6
 7include_directories(${PCL_INCLUDE_DIRS})
 8link_directories(${PCL_LIBRARY_DIRS})
 9add_definitions(${PCL_DEFINITIONS})
10
11add_executable (narf_descriptor_visualization narf_descriptor_visualization.cpp)
12target_link_libraries (narf_descriptor_visualization ${PCL_LIBRARIES})

You can now try it with a point cloud file from your hard drive:

$ ./narf_descriptor_visualization <point_cloud.pcd>

It will take a few second, during which you will see the status in the terminal. During this time, a NARF feature is extracted in every point of the range image created from the given point cloud. When it is done, a widget showing the range image pops up. Now click on a point in the range image. If it is a valid image point, three additional widgets will pop up. One visualizing the actual descriptor as a row of gray values, one showing a local range image patch of the area on which you clicked, overlaid with a star shaped pattern. Each beam corresponds to one of the cells in the descriptor. The one facing upwards to the first cell and then going clockwise. The basic intuition is, that the more the surface changes under the beam, the higher (brighter) the value of the corresponding descriptor cell. There is also one or more red beams, which mark the extracted dominant orientations of the image patch, which, together with the normal, is used to create a unique orientation for the feature coordinate frame. The last image visualizes the descriptor distances to every other point in the scene. The darker the value, the more similar the point is to the clicked image point.

The result should look similar to this:

_images/narf_descriptor_visualization.png

Also have a look at:

$ ./narf_descriptor_visualization -h

for a list of parameters.