Upgrade main pipline no lidar

README.md

@@ -17,14 +17,20 @@ For a brief recap of what it does, see section [How it works](#how-it-works)
 
 ## Software Dependencies
 
+*Note*: There is a dockerfile in order to construct a docker image that automatically complies with all the software dependencies. You can just construct it with
+
+```
+docker build . -t my_image
+```
+
 These are the used tools, make sure to install them before running the scripts.
 
- - [CUDA](https://developer.nvidia.com/cuda-downloads)
- - [OpenCV](https://opencv.org/)
- - [ETH3D Dataset-pipeline](https://github.com/ETH3D/dataset-pipeline)
- - [Pytorch](https://pytorch.org/)
- - [COLMAP](https://colmap.github.io/)
- - [PDrAW from AnafiSDK](https://developer.parrot.com/docs/pdraw/)
+ - [CUDA](https://developer.nvidia.com/cuda-downloads) (version : 10+)
+ - [OpenCV](https://opencv.org/) (version, 4.0.0+)
+ - [ETH3D Dataset-pipeline](https://github.com/ETH3D/dataset-pipeline) (version : master)
+ - [Pytorch](https://pytorch.org/) (version, 1.7.0+)
+ - [COLMAP](https://colmap.github.io/) (version : master)
+ - [PDrAW from AnafiSDK](https://developer.parrot.com/docs/pdraw/) (version : master)
 
 Apart from CUDA, which you need to install by yourself, you can use the help script `install_dependencies.sh` to install them on ubuntu 20.04.
 
@@ -738,7 +744,7 @@ This will essentially do the same thing as the script, in order to let you chang
     This will create a dataset at the folder `/path/to/dataset/` with images, depth maps in npy format, camera intrinsics and distortion in txt and yaml, pose information in the same format as KITTI odometry, and relevant metadata stored in a csv file.
 
 16. Evaluation list creation
-
+    
     Once everything is constructed, you can specify a subset of e.g. 500 frames for evaluaton.
 
     ```

main_pipeline.py

@@ -141,6 +141,7 @@ def main():
         if not (env["georef_recon"]/"images.bin").isfile():
             # GPS alignment failed, possibly because not enough GPS referenced images
             # Copy the original model without alignment
+            print("Warning, model alignment failed, the model will be normalized, and thus the depth maps too")
             thorough_model.merge_tree(env["georef_recon"])
         env["georef_recon"].merge_tree(env["georef_full_recon"])
     if args.inspect_dataset:

main_pipeline_no_lidar.py

 import numpy as np
 from wrappers import Colmap, FFMpeg, PDraw, ETH3D, PCLUtil
 from cli_utils import set_full_argparser, print_step, print_workflow
-from video_localization import localize_video, generate_GT
+from video_localization import localize_video, generate_GT, generate_GT_individual_pictures
 import meshlab_xml_writer as mxw
 import prepare_images as pi
 import prepare_workspace as pw
@@ -14,7 +14,8 @@ def main():
         print_workflow()
         return
     if args.add_new_videos:
-        args.skip_step += [1, 2, 4, 5, 6]
+        env["resume_work"] = True
+        args.skip_step += [1, 3, 4]
     if args.begin_step is not None:
         args.skip_step += list(range(args.begin_step))
     pw.check_input_folder(args.input_folder, with_lidar=False)
@@ -47,21 +48,24 @@ def main():
     i = 1
     if i not in args.skip_step:
         print_step(i, "Pictures preparation")
-        env["existing_pictures"] = pi.extract_pictures_to_workspace(**env)
+        env["individual_pictures"] = pi.extract_pictures_to_workspace(**env)
     else:
-        env["existing_pictures"] = sum((list(env["colmap_img_root"].walkfiles('*{}'.format(ext))) for ext in env["pic_ext"]), [])
+        full_paths = sum((list(env["individual_pictures_path"].walkfiles('*{}'.format(ext))) for ext in env["pic_ext"]), [])
+        env["individual_pictures"] = [path.relpath(env["colmap_img_root"]) for path in full_paths]
 
     i += 1
+    # Get already existing_videos
+    env["videos_frames_folders"] = {}
+    by_name = {v.stem: v for v in env["videos_list"]}
+    for folder in env["video_path"].walkdirs():
+        video_name = folder.basename()
+        if video_name in by_name.keys():
+            env["videos_frames_folders"][by_name[video_name]] = folder
     if i not in args.skip_step:
         print_step(i, "Extracting Videos and selecting optimal frames for a thorough scan")
-        env["videos_frames_folders"] = pi.extract_videos_to_workspace(fps=args.lowfps, **env)
-    else:
-        env["videos_frames_folders"] = {}
-        by_name = {v.stem: v for v in env["videos_list"]}
-        for folder in env["video_path"].walkdirs():
-            video_name = folder.basename()
-            if video_name in by_name.keys():
-                env["videos_frames_folders"][by_name[video_name]] = folder
+        new_video_frame_folders = pi.extract_videos_to_workspace(fps=args.lowfps, **env)
+        # Concatenate both already treated videos and newly detected videos
+        env["videos_frames_folders"] = {**env["videos_frames_folders"], **new_video_frame_folders}
     env["videos_workspaces"] = {}
     for v, frames_folder in env["videos_frames_folders"].items():
         env["videos_workspaces"][v] = pw.prepare_video_workspace(v, frames_folder, **env)
@@ -72,7 +76,10 @@ def main():
         env["thorough_recon"].makedirs_p()
         colmap.extract_features(image_list=env["video_frame_list_thorough"], more=args.more_sift_features)
         colmap.index_images(vocab_tree_output=env["indexed_vocab_tree"], vocab_tree_input=args.vocab_tree)
-        colmap.match(method="vocab_tree", vocab_tree=env["indexed_vocab_tree"], max_num_matches=env["max_num_matches"])
+        if env["match_method"] == "vocab_tree":
+            colmap.match(method="vocab_tree", vocab_tree=env["indexed_vocab_tree"], max_num_matches=env["max_num_matches"])
+        else:
+            colmap.match(method="exhaustive", max_num_matches=env["max_num_matches"])
         colmap.map(output=env["thorough_recon"], multiple_models=env["multiple_models"])
         thorough_model = pi.choose_biggest_model(env["thorough_recon"])
         colmap.adjust_bundle(thorough_model, thorough_model,
@@ -84,8 +91,9 @@ def main():
     if i not in args.skip_step:
         print_step(i, "Alignment of photogrammetric reconstruction with GPS")
         env["georef_recon"].makedirs_p()
+        env["georef_full_recon"].makedirs_p()
         colmap.align_model(output=env["georef_recon"],
-                           input=env["thorough_recon"] / "0",
+                           input=thorough_model,
                            ref_images=env["georef_frames_list"])
         if not (env["georef_recon"]/"images.bin").isfile():
             # GPS alignment failed, possibly because not enough GPS referenced images
@@ -94,10 +102,13 @@ def main():
             thorough_model.merge_tree(env["georef_recon"])
         env["georef_recon"].merge_tree(env["georef_full_recon"])
     if args.inspect_dataset:
+        print("FIRST DATASET INSPECTION")
+        print("Inspection of localisalization of frames used in thorough mapping "
+              "w.r.t Sparse reconstruction")
         colmap.export_model(output=env["georef_recon"] / "georef_sparse.ply",
                             input=env["georef_recon"])
         georef_mlp = env["georef_recon"]/"georef_recon.mlp"
-        mxw.create_project(georef_mlp, [env["georefrecon_ply"]])
+        mxw.create_project(georef_mlp, [env["georef_recon"] / "georef_sparse.ply"])
         colmap.export_model(output=env["georef_recon"],
                             input=env["georef_recon"],
                             output_type="TXT")
@@ -144,9 +155,15 @@ def main():
                                 env["max_splat_size"])
 
     if args.inspect_dataset:
+        print("SECOND DATASET INSPECTION")
+        print("Inspection of localisalization of frames used in thorough mapping "
+              "w.r.t Dense reconstruction")
         eth3d.inspect_dataset(scan_meshlab=env["aligned_mlp"],
                               colmap_model=env["georef_recon"],
                               image_path=env["colmap_img_root"])
+        print("SECOND DATASET INSPECTION")
+        print("Inspection of localisalization of frames used in thorough mapping "
+              "w.r.t Dense reconstruction and Occlusion Meshes")
         eth3d.inspect_dataset(scan_meshlab=env["aligned_mlp"],
                               colmap_model=env["georef_recon"],
                               image_path=env["colmap_img_root"],
@@ -161,10 +178,18 @@ def main():
 
             generate_GT(video_name=v, GT_already_done=video_env["GT_already_done"],
                         video_index=j+1,
+                        step_index=i,
                         num_videos=len(env["videos_to_localize"]),
-                        metadata=video_env["metadata"],
+                        metadata_path=video_env["metadata_path"],
                         global_registration_matrix=np.eye(4),
                         **video_env["output_env"], **env)
+        if env["generate_groundtruth_for_individual_images"]:
+            by_folder = pi.group_pics_by_folder(env["individual_pictures"])
+            for folder, pic_list in by_folder.items():
+                generate_GT_individual_pictures(input_colmap_model=env["georef_full_recon"],
+                                                individual_pictures_list=pic_list,
+                                                relpath=folder,
+                                                step_index=i, **env)
 
 
 if __name__ == '__main__':

videos_to_colmap.py

@@ -140,6 +140,12 @@ def register_new_cameras(metadata, device, fields, database, camera_dict):
         prior_focal_length = all(params[:num_focals] != 0)
         # For unknown focal_length, put a generic placeholder
         params[:num_focals][params[:num_focals] == 0] = w / 2
+        # If cx is not set, give the default value w/2
+        if params[num_focals] == 0:
+            params[num_focals] = w/2
+        # same for cy
+        if params[num_focals + 1] == 0:
+            params[num_focals + 1] = h/2
         # We can get less params than actual params if they are unknown. We then pad it with zeros
         db_id = database.add_camera(model_id, int(w), int(h), params, prior_focal_length=prior_focal_length)
         camera_ids.append(db_id)
@@ -213,8 +219,7 @@ def get_video_metadata(v, output_video_folder, system, generic_model='OPENCV', *
                                            width, height, framerate)
             metadata["camera_model"] = "PINHOLE"
             device = "anafi"
-        except Exception as e:
-            raise e
+        except Exception:
             # No metadata found, construct a simpler dataframe without location
             metadata = generic_metadata()
             device = "generic"

wrappers/colmap.py

@@ -46,7 +46,8 @@ class Colmap(Wrapper):
     def map(self, output, input=None, multiple_models=True, start_frame_id=None):
         options = ["mapper", "--database_path", self.db,
                    "--image_path", self.image_path,
-                   "--output_path", output]
+                   "--output_path", output,
+                   "--Mapper.ba_refine_principal_point", "1"]
         if start_frame_id is not None:
             options += ["--Mapper.init_image_id1", str(start_frame_id)]
         if not multiple_models: