Commit e9d216c4 authored by Clément Pinard's avatar Clément Pinard
Browse files

Update depth evaluation

parent f50dce63
......@@ -16,10 +16,10 @@ for sample in tqdm(engine):
previous_frame, previous_intrinsics, previous_pose = sample.get_previous_frame(displacement=0.3)
estimated_depth_map = my_model(latest_frame, previous_frame, previous_pose)
engine.finish_frame(estimated_depth_map)
mean_inference_time, output_depth_maps = engine.finalize(output_file='output.npz')
mean_inference_time, output_depth_maps = engine.finalize(output_path='output.npz')
```
You can find an example usage of this Inference Framework for SfmLearner [here](https://github.com/ClementPinard/SfmLearner-Pytorch/tree/inference-framework)
You can find an example usage of this Inference Framework for SfmLearner [here](https://github.com/ClementPinard/SfmLearner-Pytorch/tree/validation_set_constructor)
## Evaluation
......
......@@ -12,12 +12,19 @@ parser = ArgumentParser(description='Evaluate depth maps with respect to ground
parser.add_argument('--dataset_root', metavar='DIR', type=Path)
parser.add_argument('--est_depth', metavar='DIR', type=Path,
help='where the depth maps are stored, must be a 3D npy file')
parser.add_argument('--evaluation_list_path', metavar='PATH', type=Path,
parser.add_argument('--evaluation_list_path', '--eval', metavar='PATH', type=Path,
help='File with list of images to test for depth evaluation')
parser.add_argument('--flight_path_vector_list', metavar='PATH', type=Path,
parser.add_argument('--flight_path_vector_list', '--fpv', metavar='PATH', type=Path,
help='File with list of speed vectors, used to compute error wrt direction')
parser.add_argument('--scale-invariant', action='store_true',
parser.add_argument('--scale_invariant', action='store_true',
help='If selected, will rescale depth map with ratio of medians')
parser.add_argument('--min_depth', metavar='D', default=1e-2, type=float,
help='threshold below which GT is discarded')
parser.add_argument('--max_depth', metavar='D', default=250, type=float,
help='threshold above which GT is discarded')
parser.add_argument('--depth_mask', metavar='PATH', default=None, type=Path,
help='path to boolean numpy array. Should be the same size as ground truth. '
'False value will discard the corresponding pixel location for every ground truth')
coords = None
......@@ -78,9 +85,15 @@ def main():
estimated_depth = np.load(args.est_depth, allow_pickle=True)
values_df = []
assert(len(depth_paths) == len(estimated_depth))
if args.depth_mask is not None:
mask = np.load(args.depth_mask)
else:
mask = None
for filepath, fpv in tqdm(zip(depth_paths, fpv_list), total=len(fpv_list)):
GT = np.load(args.dataset_root/filepath + '.npy')
new_values = get_values(GT, estimated_depth[filepath], fpv)
new_values = get_values(GT, estimated_depth[filepath], fpv,
args.scale_invariant, mask,
args.min_depth, args.max_depth)
if new_values is not None:
values_df.append(new_values)
......@@ -89,20 +102,20 @@ def main():
values_df["log_estim"] = np.log(values_df["estim"])
values_df["diff"] = values_df["estim"] - values_df["GT"]
values_df["absdiff"] = values_df["diff"].abs()
values_df["reldiff"] = values_df["diff"] / values_df["GT"]
values_df["reldiff"] = values_df["absdiff"] / values_df["GT"]
values_df["logdiff"] = values_df["log_estim"] - values_df["log_GT"]
values_df["abslogdiff"] = values_df["logdiff"].abs()
error_names = ["AbsDiff", "StdDiff", "AbsRel", "StdRel", "AbsLog", "StdLog", "a1", "a2", "a3"]
errors = [values_df["diff"].mean(),
errors = [values_df["absdiff"].mean(),
np.sqrt(np.power(values_df["diff"], 2).mean()),
values_df["reldiff"].mean(),
np.sqrt(np.power(values_df["reldiff"], 2).mean()),
values_df["logdiff"].abs().mean(),
values_df["abslogdiff"].mean(),
np.sqrt(np.power(values_df["logdiff"], 2).mean()),
sum(values_df["logdiff"] < np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 2 * np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 3 * np.log(1.25)) / len(values_df)]
sum(values_df["abslogdiff"] < np.log(1.25)) / len(values_df),
sum(values_df["abslogdiff"] < 2 * np.log(1.25)) / len(values_df),
sum(values_df["abslogdiff"] < 3 * np.log(1.25)) / len(values_df)]
print("Results for usual metrics")
print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(*error_names))
print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".format(*errors))
......@@ -194,14 +207,12 @@ def main():
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar = fig.colorbar(pl, cax=cax)
cbar.ax.tick_params(axis='y', direction='in')
cbar.set_label('# Mean abs diff', rotation=270)
pl = axes[1].imshow(mean_log_diff_per_px.T)
axes[1].set_title("Mean Log error for each pixel")
divider = make_axes_locatable(axes[1])
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar = fig.colorbar(pl, cax=cax)
cbar.ax.tick_params(axis='y', direction='in')
cbar.set_label('# Mean abs log diff', rotation=270)
plt.tight_layout()
plt.show()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment