main_pipeline_no_lidar.py 9.72 KB
Newer Older
Clément Pinard's avatar
Clément Pinard committed
1
2
import numpy as np
from wrappers import Colmap, FFMpeg, PDraw, ETH3D, PCLUtil
3
from cli_utils import set_full_argparser, print_step, print_workflow
4
from video_localization import localize_video, generate_GT, generate_GT_individual_pictures
Clément Pinard's avatar
Clément Pinard committed
5
6
7
8
9
10
import meshlab_xml_writer as mxw
import prepare_images as pi
import prepare_workspace as pw


def main():
11
    args = set_full_argparser().parse_args()
Clément Pinard's avatar
Clément Pinard committed
12
13
14
15
16
    env = vars(args)
    if args.show_steps:
        print_workflow()
        return
    if args.add_new_videos:
17
18
        env["resume_work"] = True
        args.skip_step += [1, 3, 4]
Clément Pinard's avatar
Clément Pinard committed
19
20
21
22
23
24
    if args.begin_step is not None:
        args.skip_step += list(range(args.begin_step))
    pw.check_input_folder(args.input_folder, with_lidar=False)
    args.workspace = args.workspace.abspath()
    pw.prepare_workspace(args.workspace, env, with_lidar=False)
    colmap = Colmap(db=env["thorough_db"],
25
                    image_path=env["colmap_img_root"],
Clément Pinard's avatar
Clément Pinard committed
26
27
28
29
30
31
32
33
34
35
36
                    mask_path=env["mask_path"],
                    dense_workspace=env["dense_workspace"],
                    binary=args.colmap,
                    verbose=args.verbose,
                    logfile=args.log)
    env["colmap"] = colmap
    ffmpeg = FFMpeg(args.ffmpeg, verbose=args.verbose, logfile=args.log)
    env["ffmpeg"] = ffmpeg
    pdraw = PDraw(args.nw, verbose=args.verbose, logfile=args.log)
    env["pdraw"] = pdraw
    eth3d = ETH3D(args.eth3d, args.raw_output_folder / "Images", args.max_occlusion_depth,
37
                  verbose=args.verbose, logfile=args.log, splat_radius=args.eth3d_splat_radius)
Clément Pinard's avatar
Clément Pinard committed
38
39
40
41
42
43
44
    env["eth3d"] = eth3d
    pcl_util = PCLUtil(args.pcl_util, verbose=args.verbose, logfile=args.log)
    env["pcl_util"] = pcl_util
    env["videos_list"] = sum((list((args.input_folder/"Videos").walkfiles('*{}'.format(ext))) for ext in args.vid_ext), [])
    no_gt_folder = args.input_folder/"Videos"/"no_groundtruth"
    if no_gt_folder.isdir():
        env["videos_to_localize"] = [v for v in env["videos_list"] if not str(v).startswith(no_gt_folder)]
Clément Pinard's avatar
Clément Pinard committed
45
46
    else:
        env["videos_to_localize"] = env["videos_list"]
Clément Pinard's avatar
Clément Pinard committed
47
48
49
50

    i = 1
    if i not in args.skip_step:
        print_step(i, "Pictures preparation")
51
        env["individual_pictures"] = pi.extract_pictures_to_workspace(**env)
Clément Pinard's avatar
Clément Pinard committed
52
    else:
53
54
        full_paths = sum((list(env["individual_pictures_path"].walkfiles('*{}'.format(ext))) for ext in env["pic_ext"]), [])
        env["individual_pictures"] = [path.relpath(env["colmap_img_root"]) for path in full_paths]
Clément Pinard's avatar
Clément Pinard committed
55
56

    i += 1
57
58
59
60
61
62
63
    # Get already existing_videos
    env["videos_frames_folders"] = {}
    by_name = {v.stem: v for v in env["videos_list"]}
    for folder in env["video_path"].walkdirs():
        video_name = folder.basename()
        if video_name in by_name.keys():
            env["videos_frames_folders"][by_name[video_name]] = folder
Clément Pinard's avatar
Clément Pinard committed
64
65
    if i not in args.skip_step:
        print_step(i, "Extracting Videos and selecting optimal frames for a thorough scan")
66
67
68
        new_video_frame_folders = pi.extract_videos_to_workspace(fps=args.lowfps, **env)
        # Concatenate both already treated videos and newly detected videos
        env["videos_frames_folders"] = {**env["videos_frames_folders"], **new_video_frame_folders}
Clément Pinard's avatar
Clément Pinard committed
69
70
71
72
73
74
75
76
77
78
    env["videos_workspaces"] = {}
    for v, frames_folder in env["videos_frames_folders"].items():
        env["videos_workspaces"][v] = pw.prepare_video_workspace(v, frames_folder, **env)

    i += 1
    if i not in args.skip_step:
        print_step(i, "First thorough photogrammetry")
        env["thorough_recon"].makedirs_p()
        colmap.extract_features(image_list=env["video_frame_list_thorough"], more=args.more_sift_features)
        colmap.index_images(vocab_tree_output=env["indexed_vocab_tree"], vocab_tree_input=args.vocab_tree)
79
80
81
82
        if env["match_method"] == "vocab_tree":
            colmap.match(method="vocab_tree", vocab_tree=env["indexed_vocab_tree"], max_num_matches=env["max_num_matches"])
        else:
            colmap.match(method="exhaustive", max_num_matches=env["max_num_matches"])
Clément Pinard's avatar
Clément Pinard committed
83
84
85
        colmap.map(output=env["thorough_recon"], multiple_models=env["multiple_models"])
        thorough_model = pi.choose_biggest_model(env["thorough_recon"])
        colmap.adjust_bundle(thorough_model, thorough_model,
Clément Pinard's avatar
Clément Pinard committed
86
                             num_iter=100, refine_extra_params=True)
Clément Pinard's avatar
Clément Pinard committed
87
88
    else:
        thorough_model = pi.choose_biggest_model(env["thorough_recon"])
Clément Pinard's avatar
Clément Pinard committed
89
90
91
92
93

    i += 1
    if i not in args.skip_step:
        print_step(i, "Alignment of photogrammetric reconstruction with GPS")
        env["georef_recon"].makedirs_p()
94
        env["georef_full_recon"].makedirs_p()
Clément Pinard's avatar
Clément Pinard committed
95
        colmap.align_model(output=env["georef_recon"],
96
                           input=thorough_model,
Clément Pinard's avatar
Clément Pinard committed
97
                           ref_images=env["georef_frames_list"])
Clément Pinard's avatar
Clément Pinard committed
98
        if not (env["georef_recon"]/"images.bin").isfile():
Clément Pinard's avatar
Clément Pinard committed
99
100
            # GPS alignment failed, possibly because not enough GPS referenced images
            # Copy the original model without alignment
Clément Pinard's avatar
Clément Pinard committed
101
102
            print("Warning, model alignment failed, the model will be normalized, and thus the depth maps too")
            thorough_model.merge_tree(env["georef_recon"])
Clément Pinard's avatar
Clément Pinard committed
103
104
        env["georef_recon"].merge_tree(env["georef_full_recon"])
    if args.inspect_dataset:
105
106
107
        print("FIRST DATASET INSPECTION")
        print("Inspection of localisalization of frames used in thorough mapping "
              "w.r.t Sparse reconstruction")
Clément Pinard's avatar
Clément Pinard committed
108
109
110
        colmap.export_model(output=env["georef_recon"] / "georef_sparse.ply",
                            input=env["georef_recon"])
        georef_mlp = env["georef_recon"]/"georef_recon.mlp"
111
        mxw.create_project(georef_mlp, [env["georef_recon"] / "georef_sparse.ply"])
Clément Pinard's avatar
Clément Pinard committed
112
113
114
115
116
        colmap.export_model(output=env["georef_recon"],
                            input=env["georef_recon"],
                            output_type="TXT")
        eth3d.inspect_dataset(scan_meshlab=georef_mlp,
                              colmap_model=env["georef_recon"],
117
                              image_path=env["colmap_img_root"])
Clément Pinard's avatar
Clément Pinard committed
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

    i += 1
    if i not in args.skip_step:
        print_step(i, "Video localization with respect to reconstruction")
        for j, v in enumerate(env["videos_to_localize"]):
            print("\n\nNow working on video {} [{}/{}]".format(v, j + 1, len(env["videos_to_localize"])))
            video_env = env["videos_workspaces"][v]
            localize_video(video_name=v,
                           video_frames_folder=env["videos_frames_folders"][v],
                           step_index=i, video_index=j+1,
                           num_videos=len(env["videos_to_localize"]),
                           **video_env, **env)

    i += 1
    if i not in args.skip_step:
        print_step(i, "Full reconstruction point cloud densificitation")
        env["georef_full_recon"].makedirs_p()
        colmap.undistort(input=env["georef_full_recon"])
Clément Pinard's avatar
Clément Pinard committed
136
        colmap.dense_stereo(min_depth=env["stereo_min_depth"], max_depth=env["stereo_max_depth"])
Clément Pinard's avatar
Clément Pinard committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
        colmap.stereo_fusion(output=env["georefrecon_ply"])

    i += 1
    if i not in args.skip_step:
        print_step(i, "Reconstruction cleaning")
        filtered = env["georefrecon_ply"].stripext() + "_filtered.ply"
        pcl_util.filter_cloud(input_file=env["georefrecon_ply"],
                              output_file=filtered,
                              knn=args.SOR[0], std=args.SOR[1])
        mxw.create_project(env["aligned_mlp"], [filtered])

    i += 1
    if i not in args.skip_step:
        print_step(i, "Occlusion Mesh computing")
        colmap.delaunay_mesh(env["occlusion_ply"], input_ply=env["georefrecon_ply"])
Clément Pinard's avatar
Clément Pinard committed
152
        if args.splats:
Clément Pinard's avatar
Clément Pinard committed
153
154
155
            eth3d.create_splats(env["splats_ply"], env["georefrecon_ply"].stripext() + "_filtered.ply",
                                env["occlusion_ply"], env["splat_threshold"],
                                env["max_splat_size"])
Clément Pinard's avatar
Clément Pinard committed
156
157

    if args.inspect_dataset:
158
159
160
        print("SECOND DATASET INSPECTION")
        print("Inspection of localisalization of frames used in thorough mapping "
              "w.r.t Dense reconstruction")
Clément Pinard's avatar
Clément Pinard committed
161
162
        eth3d.inspect_dataset(scan_meshlab=env["aligned_mlp"],
                              colmap_model=env["georef_recon"],
163
                              image_path=env["colmap_img_root"])
164
165
166
        print("SECOND DATASET INSPECTION")
        print("Inspection of localisalization of frames used in thorough mapping "
              "w.r.t Dense reconstruction and Occlusion Meshes")
Clément Pinard's avatar
Clément Pinard committed
167
168
        eth3d.inspect_dataset(scan_meshlab=env["aligned_mlp"],
                              colmap_model=env["georef_recon"],
169
                              image_path=env["colmap_img_root"],
Clément Pinard's avatar
Clément Pinard committed
170
171
172
173
174
175
176
177
178
179
180
                              occlusions=env["occlusion_ply"],
                              splats=env["splats_ply"])

    i += 1
    if i not in args.skip_step:
        print_step(i, "Groud Truth generation")
        for j, v in enumerate(env["videos_to_localize"]):
            video_env = env["videos_workspaces"][v]

            generate_GT(video_name=v, GT_already_done=video_env["GT_already_done"],
                        video_index=j+1,
181
                        step_index=i,
Clément Pinard's avatar
Clément Pinard committed
182
                        num_videos=len(env["videos_to_localize"]),
183
                        metadata_path=video_env["metadata_path"],
Clément Pinard's avatar
Clément Pinard committed
184
185
                        global_registration_matrix=np.eye(4),
                        **video_env["output_env"], **env)
186
187
188
189
190
191
192
        if env["generate_groundtruth_for_individual_images"]:
            by_folder = pi.group_pics_by_folder(env["individual_pictures"])
            for folder, pic_list in by_folder.items():
                generate_GT_individual_pictures(input_colmap_model=env["georef_full_recon"],
                                                individual_pictures_list=pic_list,
                                                relpath=folder,
                                                step_index=i, **env)
Clément Pinard's avatar
Clément Pinard committed
193
194
195
196


if __name__ == '__main__':
    main()