Commit 75190401 authored by Clément Pinard's avatar Clément Pinard
Browse files

final version of depth evaluation

parent 25db1cc5
......@@ -165,7 +165,7 @@ def process_one_frame(img_path, depth_path, occ_path,
if depth_path is not None:
with gzip.open(depth_path, "rb") if compressed else open(depth_path, "rb") as f:
depth = np.frombuffer(f.read(), np.float32).reshape(h, w)
output_depth_name = dataset_output_dir / img_path.basename() + '.npy'
output_depth_name = dataset_output_dir / img_path.stem + '.npy'
downscaled_depth, viz = apply_cmap_and_resize(depth, 'rainbow', downscale)
if not interpolated:
np.save(output_depth_name, downscaled_depth)
......
......@@ -47,9 +47,23 @@ def plot_distribution(values, bins, ax, label=None, log_bins=False):
ax.set_xscale('log')
def group_quantiles(df, to_group, columns, quantiles=[0.25, 0.5, 0.75]):
if isinstance(columns, str):
columns = [columns]
grouped_df = df.groupby(by=np.round(df[to_group]))
return grouped_df[columns].quantile(quantiles).unstack()
def error_map(error_per_px):
x, y = np.stack(error_per_px.index.values, axis=-1).astype(int)
error_map = np.full((int(x.max() + 1), int(y.max() + 1)), np.NaN)
error_map[x, y] = error_per_px.values
return error_map
def main():
args = parser.parse_args()
n_bins = 10
n_bins = 4
with open(args.evaluation_list_path, 'r') as f:
depth_paths = [line[:-1] for line in f.readlines()]
fpv_list = np.loadtxt(args.flight_path_vector_list)
......@@ -64,19 +78,29 @@ def main():
values_df = pd.concat(values_df)
values_df["log_GT"] = np.log(values_df["GT"])
values_df["log_estim"] = np.log(values_df["estim"])
values_df["diff"] = np.abs(values_df["GT"] - values_df["estim"])
values_df["diff"] = values_df["estim"] - values_df["GT"]
values_df["absdiff"] = values_df["diff"].abs()
values_df["reldiff"] = values_df["diff"] / values_df["GT"]
values_df["logdiff"] = np.abs(values_df["log_GT"] - values_df["log_estim"])
values_df["logdiff"] = values_df["log_estim"] - values_df["log_GT"]
values_df["abslogdiff"] = values_df["logdiff"].abs()
error_names = ["AbsDiff", "StdDiff", "AbsRel", "StdRel", "AbsLog", "StdLog", "a1", "a2", "a3"]
errors = [values_df["diff"].mean(),
np.sqrt(np.power(values_df["diff"], 2).mean()),
values_df["reldiff"].mean(),
np.sqrt(np.power(values_df["reldiff"], 2).mean()),
values_df["logdiff"].abs().mean(),
np.sqrt(np.power(values_df["logdiff"], 2).mean()),
sum(values_df["logdiff"] < np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 2 * np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 3 * np.log(1.25)) / len(values_df)]
print("Results for usual metrics")
print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(*error_names))
print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".format(*errors))
plot = True
if plot:
def error_map(error_per_px):
error_map = np.full((int(values_df["x"].max() + 1), int(values_df["y"].max() + 1)), np.NaN)
px = np.stack(error_per_px.index.values, axis=-1).astype(int)
error_map[px[0], px[1]] = error_per_px.values
return error_map
min_gt = values_df["GT"].min()
max_gt = values_df["GT"].max()
......@@ -85,24 +109,32 @@ def main():
estim_per_GT = {}
for b1, b2 in zip(bins[:-1], bins[1:]):
per_gt = values_df[(values_df["GT"] > b1) & (values_df["GT"] < b2)]
estim_per_GT[(b1+b2)/2] = {"normal": np.histogram(per_gt["estim"]),
"log_normal": np.histogram(per_gt["log_estim"]),
estim_per_GT[(b1+b2)/2] = {"normal": np.histogram(per_gt["diff"], bins=100),
"log_normal": np.histogram(per_gt["logdiff"], bins=100),
"bins": [b1, b2]}
global_diff = np.histogram(values_df["GT"] - values_df["estim"])
global_diff = np.histogram(values_df["estim"] - values_df["GT"], bins=100)
global_log_diff = np.histogram(values_df["log_GT"] - values_df["log_estim"])
global_log_diff = np.histogram(values_df["log_estim"] - values_df["log_GT"], bins=100)
metric_per_px = values_df.groupby(by=["x", "y"]).mean()
mean_diff_per_px = error_map(metric_per_px["diff"])
mean_diff_per_px = error_map(metric_per_px["absdiff"])
mean_log_diff_per_px = error_map(metric_per_px["logdiff"])
metric_per_fpv = values_df.groupby(by=np.round(values_df["fpv_dist"])).mean()
print(metric_per_fpv)
metric_per_fpv = metric_per_fpv[metric_per_fpv.index < 1000]
diff_per_fpv = metric_per_fpv["diff"]
quantiles_per_fpv = group_quantiles(values_df[values_df["fpv_dist"] < 1000],
"fpv_dist",
["absdiff", "abslogdiff"])
quantiles_per_gt = group_quantiles(values_df, "GT", ["absdiff", "abslogdiff"])
logdiff_per_fpv = metric_per_fpv["logdiff"]
# metric_per_gt = values_df.groupby(by = np.round(values_df["GT"]))
fig, axes = plt.subplots(2, 1, sharex=True)
GT_distrib = np.histogram(values_df["GT"], bins=100)
plot_distribution(*GT_distrib, axes[0])
axes[0].set_title("Ground Truth distribution")
estim_distrib = np.histogram(values_df["estim"], bins=100)
plot_distribution(*estim_distrib, axes[1])
axes[1].set_title("depth estimation distribution")
fig, axes = plt.subplots(1, 2, sharey=True)
for i, (k, v) in enumerate(estim_per_GT.items()):
......@@ -113,9 +145,6 @@ def main():
# axes[0, 0].set_title("dstribution of estimation around GT = {:.2f}".format(k))
# axes[0, 1].set_title("dstribution of log estimation around log GT = {:.2f}".format(np.log(k)))
plt.subplots_adjust(top=0.92, bottom=0.08,
left=0.10, right=0.95,
hspace=0.25, wspace=0.35)
fig, axes = plt.subplots(2, 1)
plot_distribution(*global_diff, axes[0])
axes[0].set_title("Global difference distribution from GT")
......@@ -124,10 +153,27 @@ def main():
plt.tight_layout()
fig, axes = plt.subplots(2, 1, sharex=True)
axes[0].plot(diff_per_fpv)
axes[0].set_title("Mean abs error wrt to distance to fpv (in px)")
axes[1].plot(logdiff_per_fpv)
axes[1].set_title("Mean abs log error wrt to distance to fpv (in px)")
index = quantiles_per_fpv.index
diff_per_fpv = quantiles_per_fpv["absdiff"]
logdiff_per_fpv = quantiles_per_fpv["abslogdiff"]
axes[0].fill_between(index, diff_per_fpv[0.25], diff_per_fpv[0.75], color="cyan")
axes[0].plot(diff_per_fpv[0.5].index, diff_per_fpv[0.5])
axes[0].set_title("Error wrt to distance to fpv (in px)")
axes[1].fill_between(index, logdiff_per_fpv[0.25], logdiff_per_fpv[0.75], color="cyan")
axes[1].plot(logdiff_per_fpv[0.5])
axes[1].set_title("Log error wrt to distance to fpv (in px)")
plt.tight_layout()
fig, axes = plt.subplots(2, 1, sharex=True)
index = quantiles_per_gt.index
diff_per_gt = quantiles_per_gt["absdiff"]
logdiff_per_gt = quantiles_per_gt["abslogdiff"]
axes[0].fill_between(index, diff_per_gt[0.25], diff_per_gt[0.75], color="cyan")
axes[0].plot(diff_per_gt[0.5])
axes[0].set_title("Error wrt to distance to groundtruth depth")
axes[1].fill_between(index, logdiff_per_gt[0.25], logdiff_per_gt[0.75], color="cyan")
axes[1].plot(logdiff_per_gt[0.5])
axes[1].set_title("Log error wrt to groundtruth depth")
plt.tight_layout()
fig, axes = plt.subplots(2, 1)
......@@ -138,20 +184,6 @@ def main():
plt.tight_layout()
plt.show()
error_names = ["AbsDiff", "StdDiff", "AbsRel", "StdRel", "AbsLog", "StdLog", "a1", "a2", "a3"]
errors = [values_df["diff"].mean(),
np.sqrt(np.power(values_df["diff"], 2).mean()),
values_df["reldiff"].mean(),
np.sqrt(np.power(values_df["reldiff"], 2).mean()),
values_df["logdiff"].mean(),
np.sqrt(np.power(values_df["logdiff"], 2).mean()),
sum(values_df["logdiff"] < np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 2 * np.log(1.25)) / len(values_df),
sum(values_df["logdiff"] < 3 * np.log(1.25)) / len(values_df)]
print("Results for usual metrics")
print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(*error_names))
print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".format(*errors))
if __name__ == '__main__':
main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment