Depth Segmentation

A efficient depth segmentation base on Region growing

Main idea: merge small regions and black regions to Main regions.

The image on the left is the Color source and the right is its Depth image, which are captured by an Intel Realsense F200 camera. Note that the point which is brighter implies it is farer from the camera. However, the point may become black once it is out of camera range. The red region denotes unreliable points representing too far or too close positions, which exceed the effective distance range of the camera (out of 0.2m - 1.2m). The blue region implies that the Realsense camera cannot ensure the distance is effective due to diffuse reflection.

Color Depth
Color Depth marked

The initial segmentations with topk=5 are shown in BeforeMerging. After merging small regions and black regions, we obtain the results as shown in Segmentation. The colors are random generated to mark the segmentations in BeforeMerging and Segmentation, respectively. The black regions in Segmentation are unmarked, which means that they are either unreliable points in Segmentation or do not belong to any main segment.

BeforeMerging Segmentation
Before merge Segmentation

And the whole algorithm flow is shown as follow:
Algorithm

[Head Files]

Segmentation.hpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
#include "Common.hpp"
#include "MyDraw.hpp"

class Segmentation: public Draw
{
public:
/** @brief Constructor with parameters */
Segmentation(int width, int height, unsigned topk = 5, short t = 3);
Segmentation(Size sz, unsigned topk = 5, short t = 3);
/** @brief Segment with depth image, and show some result in color image */
void Segment(Mat& depth);
void Segment(Mat& depth, Mat& color);
/**
* @brief DFS: depth first search with recurrsive methold, but easy to stack overflow
* @param depth depth data
* @param visit marked visited point
* @param cur current point position
* @param threshold the max value between extend point
* @param v from current point, a segment point set
*/

void DFS(Mat& depth, Mat& visit, Point cur, short& threshold, PointSet& v);
/**
* @brief NonRecursive: prevent stack overflow
* @param depth depth data
* @param visit marked visited point
* @param cur current point position
* @param pSet from current point, a segment point set
*/

void NonRecursive(Mat& depth, Mat& visit, Point& cur, PointSet& pSet);
/** @brief completeDepth complete black points with right/down/right-down direction which has depth value */
void completeDepth(Mat& depth);
/** @brief clear mainRegions_, blackRegions_ and distance_ */
void clear();
/** @brief return segmentation info*/
SegmentSet& getMainRegions() {return mainRegions_;}
SegmentSet& getBlackRegions() {return blackRegions_;}
vector<double>& getDistance() {return distance_;}

private:
/**
* @brief regionMerge: merge small region and black points to Main Regions
* @param depth depth data
* @param minSim 0.0 - 1.0
*/

void regionMerge(Mat& depth, double minSim);
/**
* @brief hullBoundBox: get the each region convexHull's bounding box
* @param hull the vertices of Convex hull
* @return bounding box
*/

Rect hullBoundBox(PointSet& hull);
/**
* @brief isRegionInsideHull: determine if small or black regions belongs to one Main Region
* @param pSet small region / black region
* @param hull Convex hull
* @param minSim 0.0 - 1.0
* @return in / not in
*/

bool isRegionInsideHull(PointSet& pSet, PointSet& hull, double minSim);
bool isRegionInsideHull(PointSet& pSet, PointSet& hull, PointSet& seg, double minSim);
/** @brief randColor: generate random colors to show */
void randColor();
/** @brief drawXXX: extend from Draw */
virtual void drawRotateRect(Mat& src, RotatedRect& rr);
virtual void drawConvexHull(Mat& src, PointSet& hull, Scalar color);
virtual void drawBlack(SegmentSet& blackRegions, Mat& disp, Vec3b& color);
virtual void draw(SegmentSet& segment, Mat& disp, vector<Vec3b>& colors);
virtual void drawBoundBox(SegmentSet& segment, vector<double>& distance, Mat& color, Mat& depth);
virtual void drawRegions(SegmentSet& segment, Mat& color, Mat& depth, Mat& disp);
virtual void drawSobel(Mat& depth);

private:
/** @brief _DIRECTIONS_ eight directions to extend point */
static const vector<Point> _DIRECTIONS_;
/** @brief _DIR_ three directions (right, down, right-down) to complete point */
static const vector<Point> _DIR_;
/** @brief colors_ random color to mark segmentation */
vector<Vec3b> colors_;
/** @brief RANGE_ image size to determine which region is out-of-range */
Rect RANGE_;
/** @brief threshold_ the max value between extend point */
short threshold_;
/** @brief topk_ initial main regions number */
unsigned topk_;
/** @brief segmentation info */
SegmentSet mainRegions_;
SegmentSet blackRegions_;
vector<double> distance_;
};

MyDraw.hpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
#include "Common.hpp"

class Draw
{
public:
virtual void drawRotateRect(Mat& src, RotatedRect& rr) = 0;
virtual void drawConvexHull(Mat& src, PointSet& hull, Scalar color) = 0;
virtual void drawBlack(SegmentSet& blackRegions, Mat& disp, Vec3b& color) = 0;
virtual void draw(SegmentSet& segment, Mat& disp, vector<Vec3b>& colors) = 0;
virtual void drawBoundBox(SegmentSet& segment, vector<double>& distance, Mat& color, Mat& depth) = 0;
virtual void drawRegions(SegmentSet& segment, Mat& color, Mat& depth, Mat& disp) = 0;
virtual void drawSobel(Mat& depth)=0;

};

FileOperation.hpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
#include <vector>
#include <string>
using std::vector;
using std::string;

class FileOperation{
public:
/**
* @brief getFilename: get all files / subdirs Under dirpath
* @param dirPath
* @return subFiles
*/

virtual vector<string> getFilename(const string& dirPath) ;
/**
* @brief getFilepath: get absolute path of files / subdirs Under dirpath
* @param dirPath
* @return absolutePaths
*/

virtual vector<string> getFilepath(const string& dirPath) ;
/**
* @brief findFileName: filter out filename from absolute path
* @param path
* @return filename
*/

virtual string findFileName(const string& path) ;
/**
* @brief testExist: test if file path exists or not
* @param path
* @return Ture if Exist
*/

virtual bool testExist(const string& path) ;
};

Common.hpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#include <opencv2/opencv.hpp>

using std::string;
using std::vector;

using cv::Point;
using cv::Mat;
using cv::Size;
using cv::Rect;
using cv::Scalar;
using cv::Vec3b;
using cv::RotatedRect;

typedef vector<Point> PointSet;
typedef vector<PointSet> SegmentSet;
Contents
  1. 1. A efficient depth segmentation base on Region growing
    1. 1.1. Main idea: merge small regions and black regions to Main regions.
    2. 1.2. [Head Files]
      1. 1.2.1. Segmentation.hpp
      2. 1.2.2. MyDraw.hpp
      3. 1.2.3. FileOperation.hpp
      4. 1.2.4. Common.hpp