Refactored reconstructor, Pending doc changes

Author: tsing

src/app/App.cpp

@@ -37,27 +37,32 @@ int main(int argc, char** argv)
     std::string suffix = p.get<std::string>("format");
 
     // Initialize two file readers to load images from file
-    SLS::ImageFileProcessor rightCam("rightCam");
-    SLS::ImageFileProcessor leftCam("rightCam");
+    SLS::ImageFileProcessor rightCamProcessor("rightCamProcessor");
+    SLS::ImageFileProcessor leftCamProcessor("rightCamProcessor");
 
     // Load images
     // void loadImages( const std::string &folder, std::string prefix, size_t numDigits, size_t startIdx, std::string suffix )
-    rightCam.loadImages(rightCameraFolder, "", 4, 0,suffix);
-    leftCam.loadImages(leftCameraFolder, "", 4, 0, suffix);
+    rightCamProcessor.loadImages(rightCameraFolder, "", 4, 0,suffix);
+    leftCamProcessor.loadImages(leftCameraFolder, "", 4, 0, suffix);
 
     // Load configurations, mainly calibration parameters
-    rightCam.loadConfig(rightConfigFile);
-    leftCam.loadConfig(leftConfigFile);
+    rightCamProcessor.loadConfig(rightConfigFile);
+    leftCamProcessor.loadConfig(leftConfigFile);
 
-    // Initialize a reconstructor with the resolution of the projection to project patterns
-    SLS::ReconstructorCPU reconstruct(p.get<size_t>("width"), p.get<size_t>("height"));
+    SLS::Projector proj(p.get<size_t>("width"),p.get<size_t>("height"));
 
-    // Add cameras to reconstructor
-    reconstruct.addImageProcessor(&rightCam);
-    reconstruct.addImageProcessor(&leftCam);
+    // Generate reconstruction buckets from image processors
+    std::vector<SLS::Buckets> bucketsVec
+    {
+        rightCamProcessor.generateBuckets(proj.getWidth(), proj.getHeight(), proj.getRequiredNumFrames()),
+        leftCamProcessor.generateBuckets(proj.getWidth(), proj.getHeight(), proj.getRequiredNumFrames())
+    };
+
+    // Initialize a reconstruct
+    SLS::ReconstructorCPU reconstruct;
 
     // Run reconstructio and get the point cloud
-    auto pointCloud = reconstruct.reconstruct();
+    auto pointCloud = reconstruct.reconstruct(bucketsVec);
 
     // Get extension of output file
     auto extension = output.substr(output.find_last_of(".")+1);

src/lib/core/ImageFileProcessor.cpp

@@ -161,8 +161,10 @@ Ray ImageFileProcessor::getRay(const size_t &x, const size_t &y)
         ray.dir.w=0.0;
         ray.dir = camTransMat_*ray.dir;
         ray.dir=glm::normalize(ray.dir);
+        ray.color = getColor(x, y);
         return ray;
 }
+
 Ray ImageFileProcessor::getRay(const size_t &pixelIdx) 
 {
         glm::vec2 undistorted = undistortPixel(pixelIdx);
@@ -181,7 +183,7 @@ Ray ImageFileProcessor::getRay(const size_t &pixelIdx)
         ray.dir.w=0.0;
         ray.dir = camTransMat_*ray.dir;
         ray.dir=glm::normalize(ray.dir);
-
+        ray.color = getColor(pixelIdx);
         return ray;
 }
 
@@ -255,9 +257,7 @@ Buckets ImageFileProcessor::generateBuckets(size_t projWidth, size_t projHeight,
         }
         if (!discard) {
             auto vec2Idx = bits.to_uint_gray();
-            if (projWidth > vec2Idx.x &&
-                    projHeight > vec2Idx.y)
-                bkts[vec2Idx.x * projHeight + vec2Idx.y].push_back(i);
+            bkts[vec2Idx.x * projHeight + vec2Idx.y].push_back(getRay(i));
         }
     }
     return bkts;

src/lib/core/ImageFileProcessor.h

@@ -33,7 +33,6 @@ class ImageFileProcessor: public ImageProcessor
     size_t frameIdx_;
     std::array<cv::Mat, PARAM_COUNT> params_;
     glm::mat4 camTransMat_; //Transformation matrix for camera
-    std::vector<Ray> rayTable;
     glm::vec2 undistortPixel(const glm::vec2 &distortedPixel) const;
     glm::vec2 undistortPixel(const size_t &distortedIdx) const
     {
@@ -89,7 +88,7 @@ class ImageFileProcessor: public ImageProcessor
     const cv::Mat& getNextFrame() override;
     void undistort() override;
     void computeShadowsAndThresholds() override;
-    void setResolution (const size_t &x, const size_t &y) override {resX_ = x; resY_ = y; rayTable.resize(resX_*resY_);}
+    void setResolution (const size_t &x, const size_t &y) override {resX_ = x; resY_ = y;}
     unsigned char getWhiteThreshold(size_t i) const { return thresholds_[i];}
     Buckets generateBuckets(size_t projWidth, size_t projHeight, size_t requiredNumFrames) override;
 };

src/lib/core/ImageProcessor.h

@@ -54,7 +54,8 @@ namespace SLS {
  *
  */
 
-using Buckets = std::vector<std::vector<size_t>>;
+using Bucket = std::vector<Ray>;
+using Buckets = std::vector<Bucket>;
 
 class ImageProcessor {
 protected:
@@ -70,8 +71,7 @@ class ImageProcessor {
     cv::Mat litImage_;  //!< An all lit image contains color information of
                         //!reconstruction object
 
-    uchar
-        whiteThreshold_;  //!< Thresholds are used to filter out invalid pixel.
+    uchar whiteThreshold_;  //!< Thresholds are used to filter out invalid pixel.
 
     // If the contrast of a pixel between lit and unlit is smaller than the
     // dark threshold, the pixel is invalid.

src/lib/core/Ray.h

@@ -9,6 +9,7 @@ struct Ray
 {
     vec4 origin;
     vec4 dir;
+    vec3 color;     // RGB Color
 };
 
 /**

src/lib/core/Reconstructor.h

@@ -30,18 +30,12 @@ namespace SLS
 class Reconstructor
 {
 protected:
-    std::vector<ImageProcessor*> processors_;
-    Projector* projector_;
-    std::vector<uint> idx_;
 public:
-    Reconstructor():projector_{nullptr}{};
+    Reconstructor(){};
     virtual ~Reconstructor(){}
 
     //! Reconstruct, return a point cloud
-    virtual PointCloud reconstruct()=0;
-
-    //! Add a camera pointer to reconstructor
-    virtual void addImageProcessor(ImageProcessor* processor)=0;
+    virtual PointCloud reconstruct(const std::vector<Buckets>&)=0;
 
 };
 

src/lib/core/ReconstructorCPU.cpp

@@ -5,112 +5,21 @@
 
 namespace SLS {
 
-ReconstructorCPU::~ReconstructorCPU() { delete projector_; }
-void ReconstructorCPU::initBuckets()
-{
-    size_t x, y;
-    projector_->getSize(x, y);
-    buckets_.resize(processors_.size());
-    for (auto &b : buckets_) b.resize(x * y);
-    generateBuckets();
-}
-
-void ReconstructorCPU::addImageProcessor(ImageProcessor *processor) { processors_.push_back(processor); }
-void ReconstructorCPU::generateBuckets()
-{
-    // Generating reconstruction bucket for each camera
-    for (size_t camIdx = 0; camIdx < processors_.size(); camIdx++) {
-        const auto &cam = processors_[camIdx];
-        LOG::writeLog("Generating reconstruction bucket for \"%s\" ... \n",
-                      cam->getName().c_str());
-
-        cam->computeShadowsAndThresholds();
-
-        size_t x = 0, y = 0, xTimesY = 0;
-        cam->getResolution(x, y);
-        xTimesY = x * y;
-
-        // For each camera pixel
-        for (size_t i = 0; i < xTimesY; i++) {
-            if (!cam->queryMask(i))  // No need to process if in shadow
-                continue;
-
-            // First two frames are lit and dark frame
-            // not considered
-            cam->getNextFrame();
-            cam->getNextFrame();  // skip first two frames
-
-            Dynamic_Bitset bits(projector_->getRequiredNumFrames());
-
-            bool discard = false;
-
-            // for each frame
-            for (int bitIdx = projector_->getRequiredNumFrames() - 1;
-                 bitIdx >= 0; bitIdx--) {
-                auto frame = cam->getNextFrame();
-                auto invFrame = cam->getNextFrame();
-                unsigned char pixel = frame.at<uchar>(i % y, i / y);
-                unsigned char invPixel = invFrame.at<uchar>(i % y, i / y);
-
-                // Not considering shadow mask. But the following test should be
-                // more strict than shadow mask.
-                if (invPixel > pixel &&
-                    invPixel - pixel >=
-                        ((ImageFileProcessor *)cam)->getWhiteThreshold(i)) {
-                    // No need to do anything since the array is initialized as
-                    // all zeros
-                    bits.clearBit((size_t)bitIdx);
-                    continue;
-                }
-                else if (pixel > invPixel &&
-                         pixel - invPixel >
-                             ((ImageFileProcessor *)cam)->getWhiteThreshold(i)) {
-                    bits.setBit((size_t)bitIdx);
-                }
-                else {
-                    cam->clearMask(i);
-                    discard = true;
-                    continue;
-                }
-            }  // end for each frame
-
-            // if the pixel is valid, add to bucket.
-            if (!discard) {
-                auto vec2Idx = bits.to_uint_gray();
-                if (projector_->getWidth() > vec2Idx.x &&
-                    vec2Idx.y < projector_->getHeight()) {
-                    buckets_[camIdx]
-                            [vec2Idx.x * projector_->getHeight() + vec2Idx.y]
-                                .push_back(i);
-                }
-            }
-        }
-    }
-}
-
 std::array<glm::vec3, 2> ReconstructorCPU::intersectionOfBucket_(
-    size_t firstCameraIdx, size_t secondCameraIdx, size_t bucketIdx)
+    const Bucket& firstBucket, const Bucket& secondBucket)
 {
-    const std::vector<size_t> &firstBucket =
-        buckets_[firstCameraIdx][bucketIdx];
-    const std::vector<size_t> &secondBucket =
-        buckets_[secondCameraIdx][bucketIdx];
     // for each camera pixels in two buckets
     size_t pointCount = 0;
     glm::vec3 averagePosition(0.0);
     glm::vec3 averageColor(0.0);
-    for (const auto &cam0P : firstBucket)
-        for (const auto &cam1P : secondBucket) {
+    for (const auto &ray0 : firstBucket)
+        for (const auto &ray1 : secondBucket) {
             float dist = -1.0f;
-            glm::vec4 midP =
-                midPoint(processors_[firstCameraIdx]->getRay(cam0P),
-                         processors_[secondCameraIdx]->getRay(cam1P), dist);
+            glm::vec4 midP = midPoint(ray0, ray1, dist);
             if (dist > 0.0) {
                 pointCount++;
                 averagePosition += glm::vec3(midP);
-                averageColor += (processors_[firstCameraIdx]->getColor(cam0P) +
-                                 processors_[secondCameraIdx]->getColor(cam1P)) /
-                                2.0f;
+                averageColor += (ray0.color + ray1.color) / 2.0f;
             }
         }
     if (pointCount != 0)
@@ -121,88 +30,38 @@ std::array<glm::vec3, 2> ReconstructorCPU::intersectionOfBucket_(
 }
 
 std::array<glm::vec3, 2> ReconstructorCPU::intersectionOfBucketMinDist_(
-    size_t firstCameraIdx, size_t secondCameraIdx, size_t bucketIdx)
+    const Bucket &firstBucket, const Bucket &secondBucket)
 {
-    const std::vector<size_t> &firstBucket =
-        buckets_[firstCameraIdx][bucketIdx];
-    const std::vector<size_t> &secondBucket =
-        buckets_[secondCameraIdx][bucketIdx];
     // for each camera pixels in two buckets
     glm::vec3 minPosition(0.0);
     glm::vec3 minColor(0.0);
     float minDist = std::numeric_limits<float>::max();
-    for (const auto &cam0P : firstBucket)
-        for (const auto &cam1P : secondBucket) {
+    for (const auto &ray0 : firstBucket)
+        for (const auto &ray1 : secondBucket) {
             float dist = -1.0f;
-            glm::vec4 midP =
-                midPoint(processors_[firstCameraIdx]->getRay(cam0P),
-                         processors_[secondCameraIdx]->getRay(cam1P), dist);
+            glm::vec4 midP = midPoint(ray0, ray1, dist);
             if (dist > 0.0 && dist < minDist) {
                 minDist = dist;
                 minPosition = glm::vec3(midP);
-                minColor = (processors_[firstCameraIdx]->getColor(cam0P) +
-                                 processors_[secondCameraIdx]->getColor(cam1P)) /
-                                2.0f;
+                minColor = (ray0.color + ray1.color / 2.0f);
             }
         }
     return std::array<glm::vec3, 2>{minPosition, minColor};
 }
-PointCloud ReconstructorCPU::reconstruct()
+
+PointCloud ReconstructorCPU::reconstruct(
+    const std::vector<Buckets>& multiBuckets)
 {
     PointCloud res;
-    size_t x, y;
-    // Assumes all of the cameras has the same resolution
-    processors_[0]->getResolution(x, y);
-    initBuckets();
     LOG::startTimer();
-    for (size_t i = 0; i < buckets_[0].size(); i++) {
-        std::array<glm::vec3, 2> point = intersectionOfBucket_(0, 1, i);
+    for (size_t i = 0; i < multiBuckets[0].size(); i++) {
+        std::array<glm::vec3, 2> point =
+            intersectionOfBucket_(multiBuckets[0][i], multiBuckets[1][i]);
         if (glm::all(glm::equal(point[0], glm::vec3(0.0))) &&
             glm::all(glm::equal(point[1], glm::vec3(0.0))))
             continue;
         else
             res.pushPoint(point[0], point[1]);
-
-        /*
-        const auto &cam0bucket = buckets_[0][i];
-        const auto &cam1bucket = buckets_[1][i];
-        size_t minCam0Idx = 0;
-        size_t minCam1Idx = 0;
-
-        // When non of the buckets are empty
-        if ((!cam0bucket.empty()) && (!cam1bucket.empty())) {
-            float minDist = std::numeric_limits<float>::max();
-            glm::vec4 minMidP(0.0f);
-
-            float ptCount = 0.0;
-            glm::vec4 midPointAvg(0.0f);
-
-            for (const auto &cam0P : cam0bucket)
-                for (const auto &cam1P : cam1bucket) {
-                    float dist = -1.0f;
-
-                    auto midP = midPoint(processors_[0]->getRay(cam0P),
-                                         processors_[1]->getRay(cam1P), dist);
-                    if (dist > 0.0)  // if dist is valid
-                    {
-                        ptCount += 1.0;
-                        midPointAvg += midP;
-                        if (dist < minDist) {
-                            minDist = dist;
-                            minMidP = midP;
-                            minCam0Idx = cam0P;
-                            minCam1Idx = cam1P;
-                        }
-                    }
-                }
-            midPointAvg = midPointAvg / ptCount;
-            {
-                auto color0 = processors_[0]->getColor(minCam0Idx);
-                auto color1 = processors_[1]->getColor(minCam1Idx);
-                res.pushPoint(glm::vec3(midPointAvg), (color0 + color1) / 2.0f);
-            }
-        }
-        */
     }
     LOG::endTimer("Finished reconstruction in ");
     return res;

src/lib/core/ReconstructorCPU.h

@@ -32,10 +32,6 @@ namespace SLS {
  */
 class ReconstructorCPU : public Reconstructor {
 private:
-    std::vector<std::vector<std::vector<size_t>>>
-        buckets_;  //!< Buckets [camIdx][ProjPixelIdx][camPixelIdx]
-    void initBuckets();
-    void generateBuckets();
 
     //! Find intersection of two projector pixel
     /*! Each projector pixel contains multiple camera pixels.
@@ -50,37 +46,22 @@ class ReconstructorCPU : public Reconstructor {
      * \param bucket1 Another bucket
      * \return midpoint and color
      */
-    std::array<glm::vec3, 2> intersectionOfBucket_(size_t firstCameraIdx,
-                                                   size_t secondCameraIdx,
-                                                   size_t bucketIdx);
+    std::array<glm::vec3, 2> intersectionOfBucket_(const Bucket& firstBucket,
+                                                   const Bucket& secondBucket);
 
     /*! Similar to intersectionOfBucket_(), this function finds pair of rays
      * with minimu distance to reconstruct the depth.
      *
      * This function is not used for now since the average method yield more
      * structural result.
      */
-    std::array<glm::vec3, 2> intersectionOfBucketMinDist_(size_t firstCameraIdx,
-                                                   size_t secondCameraIdx,
-                                                   size_t bucketIdx);
+    std::array<glm::vec3, 2> intersectionOfBucketMinDist_(
+        const Bucket& firstBucket, const Bucket& secondBucket);
 
 public:
-    /*! Create a reconstructor with given projector size
-    */
-    ReconstructorCPU(const size_t projX, const size_t projY) : Reconstructor()
-    {
-        projector_ = new Projector(projX, projY);
-    }
-    ~ReconstructorCPU() override;
 
     // Interfaces
     //! Reconstruct point cloud.
-    PointCloud reconstruct() override;
-
-    /*! Add a camera to reconstructor\
-     *
-     * Accepts two camera now.
-     */
-    void addImageProcessor(ImageProcessor *processor) override;
+    PointCloud reconstruct(const std::vector<Buckets>& multiBuckets) override;
 };
 }  // namespace SLS

test/RunTest.cpp

@@ -86,10 +86,14 @@ TEST( RunCPUTest, Arch)
     LC.loadImages(L_IMGS, "", 4, 0, SUFFIX);
     LC.loadConfig(L_CFG);
 
-    SLS::ReconstructorCPU rec(W, H);
-    rec.addImageProcessor(&LC);
-    rec.addImageProcessor(&RC);
-    auto pc = rec.reconstruct();
+    SLS::Projector proj(W, H);
+    SLS::ReconstructorCPU rec;
+    auto pc = rec.reconstruct(std::vector<SLS::Buckets>{
+        LC.generateBuckets(proj.getWidth(), proj.getHeight(),
+                           proj.getRequiredNumFrames()),
+        RC.generateBuckets(proj.getWidth(), proj.getHeight(),
+                           proj.getRequiredNumFrames()),
+    });
 
     pc.exportPointCloud( O_PLY, "ply");
     pc.exportPointCloud( O_OBJ, "obj");
@@ -109,6 +113,7 @@ TEST( RunCPUTest, Alexander)
     const std::string TEST_OBJ = std::string(TEST_REF_PATH) + "/alexander.obj";
     const std::string O_PLY="alexander.ply", O_OBJ="alexander.obj";
 
+    SLS::Projector proj(W, H);
     auto RC = SLS::ImageFileProcessor("RightCamera");
     RC.loadImages(R_IMGS, "", 4, 0, SUFFIX);
     RC.loadConfig(R_CFG);
@@ -117,15 +122,16 @@ TEST( RunCPUTest, Alexander)
     LC.loadImages(L_IMGS, "", 4, 0, SUFFIX);
     LC.loadConfig(L_CFG);
 
-    SLS::ReconstructorCPU rec(W, H);
-    rec.addImageProcessor(&LC);
-    rec.addImageProcessor(&RC);
-    auto pc = rec.reconstruct();
+    SLS::ReconstructorCPU rec;
+    auto pc = rec.reconstruct(std::vector<SLS::Buckets>{
+        LC.generateBuckets(proj.getWidth(), proj.getHeight(),
+                           proj.getRequiredNumFrames()),
+        RC.generateBuckets(proj.getWidth(), proj.getHeight(),
+                           proj.getRequiredNumFrames()),
+    });
 
     pc.exportPointCloud( O_PLY, "ply");
     pc.exportPointCloud( O_OBJ, "obj");
     EXPECT_TRUE(compareObjFiles(TEST_OBJ, O_OBJ));
     EXPECT_TRUE(comparePlyFiles(TEST_PLY, TEST_PLY));
 }
-
-