Commit b3288306 authored by Clément Pinard's avatar Clément Pinard
Browse files

Make depth toolkit a package

parent 75190401
...@@ -3,9 +3,10 @@ from path import Path ...@@ -3,9 +3,10 @@ from path import Path
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
from tqdm import tqdm from tqdm import tqdm
parser = ArgumentParser(description='Convert EuroC dataset to COLMAP', parser = ArgumentParser(description='Evaluate depth maps with respect to ground truth depth and FPV position',
formatter_class=ArgumentDefaultsHelpFormatter) formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('--dataset_root', metavar='DIR', type=Path) parser.add_argument('--dataset_root', metavar='DIR', type=Path)
...@@ -21,16 +22,23 @@ parser.add_argument('--scale-invariant', action='store_true', ...@@ -21,16 +22,23 @@ parser.add_argument('--scale-invariant', action='store_true',
coords = None coords = None
def get_values(gt_depth, estim_depth, fpv, min_depth=1e-2, max_depth=250): def get_values(gt_depth, estim_depth, fpv, scale_invariant=False, mask=None, min_depth=1e-2, max_depth=250):
global coords global coords
if coords is None: if coords is None:
coords = np.stack(np.meshgrid(np.arange(gt_depth.shape[1]), np.arange(gt_depth.shape[0])), axis=-1) coords = np.stack(np.meshgrid(np.arange(gt_depth.shape[1]), np.arange(gt_depth.shape[0])), axis=-1)
fpv_dist = np.linalg.norm(coords - fpv, axis=-1) fpv_dist = np.linalg.norm(coords - fpv, axis=-1)
estim_depth = np.clip(estim_depth, min_depth, max_depth) estim_depth = np.clip(estim_depth, min_depth, max_depth)
valid = (gt_depth > min_depth) & (gt_depth < max_depth) valid = (gt_depth > min_depth) & (gt_depth < max_depth)
if mask is not None:
valid = valid & mask
if valid.sum() == 0:
return
valid_gt, valid_estim = gt_depth[valid], estim_depth[valid]
if scale_invariant:
valid_estim = valid_estim * np.median(valid_gt) / np.median(valid_estim)
fpv_dist = fpv_dist[valid] fpv_dist = fpv_dist[valid]
valid_coords = coords[valid] valid_coords = coords[valid]
values = np.stack([gt_depth[valid], estim_depth[valid], *valid_coords.T, fpv_dist], axis=-1) values = np.stack([valid_gt, valid_estim, *valid_coords.T, fpv_dist], axis=-1)
return pd.DataFrame(values, columns=["GT", "estim", "x", "y", "fpv_dist"]) return pd.DataFrame(values, columns=["GT", "estim", "x", "y", "fpv_dist"])
...@@ -70,10 +78,11 @@ def main(): ...@@ -70,10 +78,11 @@ def main():
estimated_depth = np.load(args.est_depth, allow_pickle=True) estimated_depth = np.load(args.est_depth, allow_pickle=True)
values_df = [] values_df = []
assert(len(depth_paths) == len(estimated_depth)) assert(len(depth_paths) == len(estimated_depth))
for filepath, fpv in zip(depth_paths, tqdm(fpv_list)): for filepath, fpv in tqdm(zip(depth_paths, fpv_list), total=len(fpv_list)):
GT = np.load(args.dataset_root/filepath + '.npy') GT = np.load(args.dataset_root/filepath + '.npy')
new_values = get_values(GT, estimated_depth[filepath], fpv) new_values = get_values(GT, estimated_depth[filepath], fpv)
values_df.append(new_values) if new_values is not None:
values_df.append(new_values)
values_df = pd.concat(values_df) values_df = pd.concat(values_df)
values_df["log_GT"] = np.log(values_df["GT"]) values_df["log_GT"] = np.log(values_df["GT"])
...@@ -156,31 +165,43 @@ def main(): ...@@ -156,31 +165,43 @@ def main():
index = quantiles_per_fpv.index index = quantiles_per_fpv.index
diff_per_fpv = quantiles_per_fpv["absdiff"] diff_per_fpv = quantiles_per_fpv["absdiff"]
logdiff_per_fpv = quantiles_per_fpv["abslogdiff"] logdiff_per_fpv = quantiles_per_fpv["abslogdiff"]
axes[0].fill_between(index, diff_per_fpv[0.25], diff_per_fpv[0.75], color="cyan") axes[0].fill_between(index, diff_per_fpv[0.25], diff_per_fpv[0.75], color="cyan", label="25% - 75%")
axes[0].plot(diff_per_fpv[0.5].index, diff_per_fpv[0.5]) axes[0].plot(diff_per_fpv[0.5].index, diff_per_fpv[0.5], label="median")
axes[0].set_title("Error wrt to distance to fpv (in px)") axes[0].set_title("Error wrt to distance to fpv (in px)")
axes[1].fill_between(index, logdiff_per_fpv[0.25], logdiff_per_fpv[0.75], color="cyan") axes[1].fill_between(index, logdiff_per_fpv[0.25], logdiff_per_fpv[0.75], color="cyan", label="25% - 75%")
axes[1].plot(logdiff_per_fpv[0.5]) axes[1].plot(logdiff_per_fpv[0.5], label="median")
axes[1].set_title("Log error wrt to distance to fpv (in px)") axes[1].set_title("Log error wrt to distance to fpv (in px)")
axes[1].set_xlabel("Distance to flight path vector (in px)")
plt.tight_layout() plt.tight_layout()
fig, axes = plt.subplots(2, 1, sharex=True) fig, axes = plt.subplots(2, 1, sharex=True)
index = quantiles_per_gt.index index = quantiles_per_gt.index
diff_per_gt = quantiles_per_gt["absdiff"] diff_per_gt = quantiles_per_gt["absdiff"]
logdiff_per_gt = quantiles_per_gt["abslogdiff"] logdiff_per_gt = quantiles_per_gt["abslogdiff"]
axes[0].fill_between(index, diff_per_gt[0.25], diff_per_gt[0.75], color="cyan") axes[0].fill_between(index, diff_per_gt[0.25], diff_per_gt[0.75], color="cyan", label="25% - 75%")
axes[0].plot(diff_per_gt[0.5]) axes[0].plot(diff_per_gt[0.5], label="median")
axes[0].set_title("Error wrt to distance to groundtruth depth") axes[0].set_title("Error wrt to distance to groundtruth depth")
axes[1].fill_between(index, logdiff_per_gt[0.25], logdiff_per_gt[0.75], color="cyan") axes[1].fill_between(index, logdiff_per_gt[0.25], logdiff_per_gt[0.75], color="cyan", label="25% - 75%")
axes[1].plot(logdiff_per_gt[0.5]) axes[1].plot(logdiff_per_gt[0.5], label="median")
axes[1].set_title("Log error wrt to groundtruth depth") axes[1].set_title("Log error wrt to groundtruth depth")
axes[1].set_xlabel("Groundtruth depth (in meters)")
plt.tight_layout() plt.tight_layout()
fig, axes = plt.subplots(2, 1) fig, axes = plt.subplots(2, 1)
axes[0].imshow(mean_diff_per_px.T) pl = axes[0].imshow(mean_diff_per_px.T)
axes[0].set_title("Mean error for each pixel") axes[0].set_title("Mean error for each pixel")
axes[1].imshow(mean_log_diff_per_px.T) divider = make_axes_locatable(axes[0])
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar = fig.colorbar(pl, cax=cax)
cbar.ax.tick_params(axis='y', direction='in')
cbar.set_label('# Mean abs diff', rotation=270)
pl = axes[1].imshow(mean_log_diff_per_px.T)
axes[1].set_title("Mean Log error for each pixel") axes[1].set_title("Mean Log error for each pixel")
divider = make_axes_locatable(axes[1])
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar = fig.colorbar(pl, cax=cax)
cbar.ax.tick_params(axis='y', direction='in')
cbar.set_label('# Mean abs log diff', rotation=270)
plt.tight_layout() plt.tight_layout()
plt.show() plt.show()
......
...@@ -44,19 +44,31 @@ class inferenceFramework(object): ...@@ -44,19 +44,31 @@ class inferenceFramework(object):
self.min_depth, self.max_depth = min_depth, max_depth self.min_depth, self.max_depth = min_depth, max_depth
self.max_shift = max_shift self.max_shift = max_shift
self.frame_transform = frame_transform self.frame_transform = frame_transform
self.inference_time = []
self.estimated_depth_maps = {}
def __getitem__(self, i): def __getitem__(self, i):
timer = Timer() timer = Timer()
sample = inferenceSample(self.root, self.test_files[i], self.max_shift, timer, self.frame_transform) self.i = i
sample.timer.start() self.current_sample = inferenceSample(self.root, self.test_files[i], self.max_shift, timer, self.frame_transform)
return sample self.current_sample.timer.start()
return self.current_sample
def finish_frame(self, sample):
sample.timer.stop() def finish_frame(self, estimated_depth):
return sample.timer.get_elapsed() self.current_sample.timer.stop()
elapsed = self.current_sample.timer.get_elapsed()
self.inference_time.append(elapsed)
self.estimated_depth_maps[self.current_sample.file] = estimated_depth
return elapsed
def finalize(self, output_path=None):
if output_path is not None:
np.savez(output_path, **self.estimated_depth_maps)
mean_inference_time = np.mean(self.inference_time)
return mean_inference_time, self.estimated_depth_maps
def __len__(self): def __len__(self):
return len(self.img_files) return len(self.test_files)
class inferenceSample(object): class inferenceSample(object):
...@@ -148,17 +160,13 @@ def inference_toolkit_example(): ...@@ -148,17 +160,13 @@ def inference_toolkit_example():
# return np.exp(np.random.randn(frame.shape[0], frame.shape[1])) # return np.exp(np.random.randn(frame.shape[0], frame.shape[1]))
engine = inferenceFramework(args.dataset_root, evaluation_list, lambda x: x.transpose(2, 0, 1).astype(np.float32)[None]/255) engine = inferenceFramework(args.dataset_root, evaluation_list, lambda x: x.transpose(2, 0, 1).astype(np.float32)[None]/255)
estimated_depth_maps = {} for sample in tqdm(engine):
mean_time = []
for sample, image_path in zip(engine, tqdm(evaluation_list)):
latest_frame, latest_intrinsics, _ = sample.get_frame() latest_frame, latest_intrinsics, _ = sample.get_frame()
previous_frame, previous_intrinsics, previous_pose = sample.get_previous_frame(displacement=0.3) previous_frame, previous_intrinsics, previous_pose = sample.get_previous_frame(displacement=0.3)
estimated_depth_maps[image_path] = (my_model(latest_frame, previous_frame, previous_pose)) engine.finish_frame(my_model(latest_frame, previous_frame, previous_pose))
time_spent = engine.finish_frame(sample)
mean_time.append(time_spent)
print("Mean time per sample : {:.2f}us".format(1e6 * sum(mean_time)/len(mean_time))) mean_time, _ = engine.finalize(args.depth_output)
np.savez(args.depth_output, **estimated_depth_maps) print("Mean time per sample : {:.2f}us".format(1e6 * mean_time))
if __name__ == '__main__': if __name__ == '__main__':
......
from setuptools import setup, find_packages
with open("README.md", "r") as fh:
long_description = fh.read()
setup(name='inference toolkit',
license='MIT',
author='Clément Pinard',
author_email='clempinard@gmail.com',
description='Inference and evaluation routines to test on a dataset constructed with validation set constructor',
long_description=long_description,
long_description_content_type="text/markdown",
packages=find_packages(),
entry_points={
'console_scripts': [
'depth_evaluation = evaluation_toolkit.depth_evaluation:main'
]
},
install_requires=[
'numpy',
'pandas',
'path',
'imageio',
'scikit-image',
'scipy',
'tqdm'
],
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License",
"Intended Audience :: Science/Research"
]
)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment