Commit 44ca0766 authored by Clément Pinard's avatar Clément Pinard
Browse files

add inference and evaluation toolkit

parent 8f5dfd03
......@@ -65,6 +65,7 @@ def main():
valid_frames = s.iloc[args.min_shift:]
valid_frames = valid_frames[~valid_frames["interpolated"]]
valid_frames["image_path"] = [args.dataset_dir.relpathto(v) / Path(f).basename() for f in valid_frames["image_path"].values]
total_valid_frames.append(valid_frames)
total_valid_frames_df = pd.concat(total_valid_frames)
......
......@@ -13,6 +13,7 @@ import gzip
from pebble import ProcessPool
import yaml
from itertools import product
import pandas as pd
def rescale_and_save_cameras(cameras, images, output_dir, output_width=None, downscale=None):
......@@ -91,6 +92,7 @@ def save_poses(images, images_list, output_dir):
relative_position = np.linalg.inv(starting_pos) @ current_pos
poses.append(relative_position[:3])
except KeyError:
# Frame is not registered so we put NaN coordinates instead
poses.append(np.full((3, 4), np.NaN))
poses = np.stack(poses)
np.savetxt(output_dir/'poses.txt', poses.reshape((len(images_list), -1)))
......@@ -225,7 +227,7 @@ def convert_dataset(final_model, depth_dir, images_root_folder, occ_dir,
video_output_dir.makedirs_p()
if video:
visualization = True
cameras, images, _ = rm.read_model(final_model, '.txt')
cameras_colmap, images_colmap, _ = rm.read_model(final_model, '.txt')
# image_df = pd.DataFrame.from_dict(images, orient="index").set_index("id")
if metadata is not None:
......@@ -239,14 +241,17 @@ def convert_dataset(final_model, depth_dir, images_root_folder, occ_dir,
video = False
# Discard images and cameras that are not represented by the image list
images = {i.name: i for k, i in images.items() if i.name in images_list}
cameras_ids = set([i.camera_id for i in images.values()])
cameras = {k: cameras[k] for k in cameras_ids}
images_colmap = {i.name: i for k, i in images_colmap.items() if i.name in images_list}
cameras_ids = set([i.camera_id for i in images_colmap.values()])
cameras_colmap = {k: cameras_colmap[k] for k in cameras_ids}
if downscale is None:
assert width is not None
rescaled_cameras = rescale_and_save_cameras(cameras, images, dataset_output_dir, width, downscale)
poses = save_poses(images, images_list, dataset_output_dir)
rescaled_cameras = rescale_and_save_cameras(cameras_colmap,
images_colmap,
dataset_output_dir,
width, downscale)
poses = save_poses(images_colmap, images_list, dataset_output_dir)
depth_maps = []
occ_maps = []
......@@ -264,7 +269,8 @@ def convert_dataset(final_model, depth_dir, images_root_folder, occ_dir,
if compressed:
depth_path += ".gz"
occ_path += ".gz"
if depth_path.isfile():
if i in images_colmap:
assert depth_path.isfile()
registered.append(True)
if occ_path.isfile():
occ_maps.append(occ_path)
......@@ -349,6 +355,7 @@ def convert_dataset(final_model, depth_dir, images_root_folder, occ_dir,
if __name__ == '__main__':
args = parser.parse_args()
env = vars(args)
env["metadata"] = pd.read_csv(env["metadata_path"])
if args.interpolated_frames_path is None:
env["interpolated_frames"] = []
else:
......
......@@ -3,6 +3,7 @@ from path import Path
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
parser = ArgumentParser(description='Convert EuroC dataset to COLMAP',
formatter_class=ArgumentDefaultsHelpFormatter)
......@@ -10,55 +11,57 @@ parser = ArgumentParser(description='Convert EuroC dataset to COLMAP',
parser.add_argument('--dataset_root', metavar='DIR', type=Path)
parser.add_argument('--est_depth', metavar='DIR', type=Path,
help='where the depth maps are stored, must be a 3D npy file')
parser.add_argument('--evaluation_list', metavar='PATH', type=Path,
parser.add_argument('--evaluation_list_path', metavar='PATH', type=Path,
help='File with list of images to test for depth evaluation')
parser.add_argument('--flight_path_vector_list', metavar='PATH', type=Path,
help='File with list of speed vectors, used to compute error wrt direction')
parser.add_argument('--scale-invariant', action='store_true',
help='If selected, will rescale depth map with ratio of medians')
coord = None
coords = None
def get_values(gt_depth, estim_depth, fpv):
def get_values(gt_depth, estim_depth, fpv, min_depth=1e-2, max_depth=250):
global coords
if coords is None:
coords = np.stack(np.meshgrid(np.arange(gt_depth.shape[0]), np.arange(gt_depth.shape[1])), axis=-1)
coords = np.stack(np.meshgrid(np.arange(gt_depth.shape[1]), np.arange(gt_depth.shape[0])), axis=-1)
fpv_dist = np.linalg.norm(coords - fpv, axis=-1)
valid = gt_depth == gt_depth & gt_depth < np.inf
estim_depth = np.clip(estim_depth, min_depth, max_depth)
valid = (gt_depth > min_depth) & (gt_depth < max_depth)
fpv_dist = fpv_dist[valid]
valid_coords = coords[valid[..., None]]
valid_coords = coords[valid]
values = np.stack([gt_depth[valid], estim_depth[valid], *valid_coords.T, fpv_dist], axis=-1)
return pd.DataFrame(values, columns=["GT", "estim", "x", "y", "fpv_dist"])
def plot_distribution(bins, values, ax):
def plot_distribution(values, bins, ax, label=None, log_bins=False):
bin_dists = bins[1:] - bins[:-1]
total = sum(bin_dists)
normalized_values = values * bin_dists / total
normalized_values = (values / sum(values)) * bin_dists / total
bin_centers = 0.5*(bins[1:] + bins[:-1])
ax.plot(bin_centers, normalized_values)
if log_bins:
bin_centers = np.exp(bin_centers)
ax.plot(bin_centers, normalized_values, label=label)
if log_bins:
ax.set_xscale('log')
def main():
args = parser.parse_args()
n_bins = 10
with open(args.evaluation_list, 'r') as f:
with open(args.evaluation_list_path, 'r') as f:
depth_paths = [line[:-1] for line in f.readlines()]
fpv_list = np.readtxt(args.flight_path_vector_list)
estimated_depth = np.load(args.est_depth)
values_df = None
assert(len(depth_paths) == estimated_depth.shape[0])
for filepath, current_estimated_depth, fpv in zip(depth_paths, estimated_depth, fpv_list):
GT = np.load(filepath)
new_values = get_values(GT, current_estimated_depth, fpv)
if values_df is None:
values_df = new_values
else:
values_df = values_df.append(new_values)
fpv_list = np.loadtxt(args.flight_path_vector_list)
estimated_depth = np.load(args.est_depth, allow_pickle=True)
values_df = []
assert(len(depth_paths) == len(estimated_depth))
for filepath, fpv in zip(depth_paths, tqdm(fpv_list)):
GT = np.load(args.dataset_root/filepath + '.npy')
new_values = get_values(GT, estimated_depth[filepath], fpv)
values_df.append(new_values)
values_df = pd.concat(values_df)
values_df["log_GT"] = np.log(values_df["GT"])
values_df["log_estim"] = np.log(values_df["estim"])
values_df["diff"] = np.abs(values_df["GT"] - values_df["estim"])
......@@ -68,10 +71,10 @@ def main():
plot = True
if plot:
def error_map(series):
error_per_px = series.groupby(by=["x", "y"]).mean()
error_map = np.full(estimated_depth.shape[:2], np.NaN)
error_map[error_per_px.index] = error_per_px.values
def error_map(error_per_px):
error_map = np.full((int(values_df["x"].max() + 1), int(values_df["y"].max() + 1)), np.NaN)
px = np.stack(error_per_px.index.values, axis=-1).astype(int)
error_map[px[0], px[1]] = error_per_px.values
return error_map
min_gt = values_df["GT"].min()
......@@ -81,51 +84,58 @@ def main():
estim_per_GT = {}
for b1, b2 in zip(bins[:-1], bins[1:]):
per_gt = values_df[values_df["GT"] > b1 & values_df["GT"] < b2]
per_gt = values_df[(values_df["GT"] > b1) & (values_df["GT"] < b2)]
estim_per_GT[(b1+b2)/2] = {"normal": np.histogram(per_gt["estim"]),
"log_normal": np.histogram(per_gt["log_estim"])}
"log_normal": np.histogram(per_gt["log_estim"]),
"bins": [b1, b2]}
global_diff = np.histogram(values_df["GT"] - values_df["estim"])
global_log_diff = np.histogram(values_df["log_GT"] - values_df["log_estim"])
mean_diff_per_px = error_map(values_df["diff"])
mean_log_diff_per_px = error_map(values_df["logdiff"])
per_fpv = values_df["diff"].groupby(by=np.round(["fpv_dist"])).mean()
metric_per_px = values_df.groupby(by=["x", "y"]).mean()
mean_diff_per_px = error_map(metric_per_px["diff"])
mean_log_diff_per_px = error_map(metric_per_px["logdiff"])
log_diff_per_px = values_df["logdiff"].groupby(by=["x", "y"]).mean()
log_error_map = np.full(estimated_depth.shape[:2], np.NaN)
log_error_map[log_diff_per_px.index] = log_diff_per_px
metric_per_fpv = values_df.groupby(by=np.round(values_df["fpv_dist"])).mean()
print(metric_per_fpv)
metric_per_fpv = metric_per_fpv[metric_per_fpv.index < 1000]
diff_per_fpv = metric_per_fpv["diff"]
log_per_fpv = values_df["logdiff"].groupby(by=np.round(["fpv_dist"])).mean()
logdiff_per_fpv = metric_per_fpv["logdiff"]
fig, axes = plt.subplots(len(estim_per_GT), 2, figsize=(15, 20), dpi=200)
fig, axes = plt.subplots(1, 2, sharey=True)
for i, (k, v) in enumerate(estim_per_GT.items()):
plot_distribution(axes[i, 0], v["normal"])
plot_distribution(axes[i, 1], v["log_normal"])
axes[i, 0].set_title("dstribution of estimation around GT = {:.2f}".format(k))
axes[i, 1].set_title("dstribution of log estimation around log GT = {:.2f}".format(np.log(k)))
fig, axes = plt.subplots(2, 1, figsize=(15, 20), dpi=200)
plot_distribution(axes[0, 0], global_diff)
axes[0, 0].set_title("Global difference distribution from GT")
plot_distribution(axes[1, 0], global_log_diff)
axes[1, 0].set_title("Global log difference distribution from GT")
fig, axes = plt.subplots(2, 1, figsize=(15, 20), dpi=200)
plot_distribution(axes[0, 0], per_fpv)
axes[0, 0].set_title("Mean abs error wrt to distance to fpv (in px)")
plot_distribution(axes[1, 0], log_per_fpv)
axes[0, 0].set_title("Mean abs log error wrt to distance to fpv (in px)")
fig, axes = plt.subplots(2, 1, figsize=(15, 20), dpi=200)
axes[0, 0].imshow(mean_diff_per_px)
axes[0, 0].set_title("Mean error for each pixel")
axes[1, 0].imshow(mean_log_diff_per_px)
axes[1, 0].set_title("Mean Log error for each pixel")
plot_distribution(*v["normal"], axes[0], label="$GT \\in [{:.1f},{:.1f}]$".format(*v["bins"]))
plot_distribution(*v["log_normal"], axes[1], label="$GT \\in [{:.1f},{:.1f}]$".format(*v["bins"]), log_bins=True)
axes[0].legend()
axes[1].legend()
# axes[0, 0].set_title("dstribution of estimation around GT = {:.2f}".format(k))
# axes[0, 1].set_title("dstribution of log estimation around log GT = {:.2f}".format(np.log(k)))
plt.subplots_adjust(top=0.92, bottom=0.08,
left=0.10, right=0.95,
hspace=0.25, wspace=0.35)
fig, axes = plt.subplots(2, 1)
plot_distribution(*global_diff, axes[0])
axes[0].set_title("Global difference distribution from GT")
plot_distribution(*global_log_diff, axes[1], log_bins=True)
axes[1].set_title("Global log difference distribution from GT")
plt.tight_layout()
fig, axes = plt.subplots(2, 1, sharex=True)
axes[0].plot(diff_per_fpv)
axes[0].set_title("Mean abs error wrt to distance to fpv (in px)")
axes[1].plot(logdiff_per_fpv)
axes[1].set_title("Mean abs log error wrt to distance to fpv (in px)")
plt.tight_layout()
fig, axes = plt.subplots(2, 1)
axes[0].imshow(mean_diff_per_px.T)
axes[0].set_title("Mean error for each pixel")
axes[1].imshow(mean_log_diff_per_px.T)
axes[1].set_title("Mean Log error for each pixel")
plt.tight_layout()
plt.show()
error_names = ["AbsDiff", "StdDiff", "AbsRel", "StdRel", "AbsLog", "StdLog", "a1", "a2", "a3"]
......@@ -134,9 +144,10 @@ def main():
values_df["reldiff"].mean(),
np.sqrt(np.power(values_df["reldiff"], 2).mean()),
values_df["logdiff"].mean(),
sum(values_df["log_diff"] < np.log(1.25)) / len(values_df),
sum(values_df["log_diff"] < 2 * np.log(1.25)) / len(values_df),
sum(values_df["log_diff"] < 3 * np.log(1.25)) / len(values_df)]
np.sqrt(np.power(values_df["logdiff"], 2).mean()),
sum(values_df["logdiff"] < np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 2 * np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 3 * np.log(1.25)) / len(values_df)]
print("Results for usual metrics")
print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(*error_names))
print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".format(*errors))
......
......@@ -3,6 +3,8 @@ from path import Path
from imageio import imread
import time
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from scipy.spatial.transform import Rotation
from tqdm import tqdm
class Timer:
......@@ -38,10 +40,11 @@ class inferenceFramework(object):
self.test_files = test_files
self.min_depth, self.max_depth = min_depth, max_depth
self.max_shift = max_shift
self.frame_transform = frame_transform
def __getitem__(self, i):
timer = Timer()
sample = inferenceSample(self.root, self.test_files[i], timer, self.max_shift, self.frame_transform)
sample = inferenceSample(self.root, self.test_files[i], self.max_shift, timer, self.frame_transform)
sample.timer.start()
return sample
......@@ -58,23 +61,22 @@ class inferenceSample(object):
self.root = root
self.file = file
self.frame_transform = frame_transform
self.timer = timer
full_filepath = self.root / file
scene = full_filepath.parent
scene_files = sorted(scene.files("*jpg"))
scene_files = sorted(scene.files("*.jpg"))
poses = np.genfromtxt(scene / "poses.txt").reshape((-1, 3, 4))
sample_id = scene_files.index(full_filepath)
assert(sample_id > max_shift)
start_id = sample_id - max_shift
self.valid_frames = scene_files[start_id:sample_id + 1][::-1]
valid_poses = poses[start_id:sample_id + 1].flipud()
valid_poses_full = np.concatenate([valid_poses, np.array([0, 0, 0, 1]).reshape(1, 4, 1)])
valid_poses = np.flipud(poses[start_id:sample_id + 1])
last_line = np.broadcast_to(np.array([0, 0, 0, 1]), (valid_poses.shape[0], 1, 4))
valid_poses_full = np.concatenate([valid_poses, last_line], axis=1)
self.poses = (np.linalg.inv(valid_poses_full[0]) @ valid_poses_full)[:, :3]
R = self.poses[:, :3, :3]
s = np.linalg.norm(np.stack([R[:, 0, 1]-R[:, 1, 0],
R[:, 1, 2]-R[:, 2, 1],
R[:, 0, 2]-R[:, 2, 0]]), axis=1)
self.rotation_angles = np.abs(np.arcsin(0.5 * s))
self.displacements = np.linalg.norm(self.poses[:, :, 4])
self.rotation_angles = Rotation.from_matrix(R).magnitude()
self.displacements = np.linalg.norm(self.poses[:, :, -1])
if (scene / "intrinsics.txt").isfile():
self.intrinsics = np.stack([np.genfromtxt(scene / "intrinsics.txt")]*max_shift)
......@@ -95,15 +97,13 @@ class inferenceSample(object):
self.timer.stop()
if displacement is not None:
shift = (self.poses[:, :, -1] - displacement).argmin()
shift = max(1, (self.displacements - displacement).argmin())
rot_valid = self.rotation_angles < max_rot
assert sum(rot_valid[1:shift] > 0), "Rotation is alaways higher than {}".format(max_rot)
assert sum(rot_valid[1:shift+1] > 0), "Rotation is alaways higher than {}".format(max_rot)
# Highest shift that has rotation below max_rot thresold
final_shift = np.where(rot_valid[-1 - shift:])[-1]
final_shift = np.where(rot_valid[-1 - shift:])[0][-1]
self.timer.start()
return *self.get_frame(final_shift), self.poses[final_shift]
return self.get_frame(final_shift)
def inference_toolkit_example():
......@@ -111,7 +111,7 @@ def inference_toolkit_example():
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--dataset_root', metavar='DIR', type=Path)
parser.add_argument('--depth_output', metavar='DIR', type=Path,
parser.add_argument('--depth_output', metavar='FILE', type=Path,
help='where to store the estimated depth maps, must be a npy file')
parser.add_argument('--evaluation_list_path', metavar='PATH', type=Path,
help='File with list of images to test for depth evaluation')
......@@ -124,15 +124,16 @@ def inference_toolkit_example():
def my_model(frame, previous, pose):
# Mock up function that uses two frames and translation magnitude
return np.linalg.norm(pose[:, -1]) * np.linalg.norm(frame - previous, axis=-1)
# return np.linalg.norm(pose[:, -1]) * np.linalg.norm(frame - previous, axis=-1)
return np.exp(np.random.randn(frame.shape[0], frame.shape[1]))
engine = inferenceFramework(args.root, evaluation_list, lambda x: x.transpose(2, 0, 1).astype(np.float32)[None]/255)
engine = inferenceFramework(args.dataset_root, evaluation_list, lambda x: x.transpose(2, 0, 1).astype(np.float32)[None]/255)
esimated_depth_maps = {}
mean_time = []
for sample, image_path in zip(engine, evaluation_list):
for sample, image_path in zip(engine, tqdm(evaluation_list)):
latest_frame, latest_intrinsics, _ = sample.get_frame()
previous_frame, previous_intrinsics, previous_pose = sample.get_previous_frame(displacement=0.3)
esimated_depth_maps[image_path] = (my_model(latest_frame, previous_frame))
esimated_depth_maps[image_path] = (my_model(latest_frame, previous_frame, previous_pose))
time_spent = engine.finish_frame(sample)
mean_time.append(time_spent)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment