depth_evaluation.py 9.77 KB
Newer Older
1
2
3
4
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from path import Path
import numpy as np
import pandas as pd
5
import matplotlib.pyplot as plt
Clément Pinard's avatar
Clément Pinard committed
6
from mpl_toolkits.axes_grid1 import make_axes_locatable
7
from tqdm import tqdm
8

Clément Pinard's avatar
Clément Pinard committed
9
parser = ArgumentParser(description='Evaluate depth maps with respect to ground truth depth and FPV position',
10
11
12
13
14
                        formatter_class=ArgumentDefaultsHelpFormatter)

parser.add_argument('--dataset_root', metavar='DIR', type=Path)
parser.add_argument('--est_depth', metavar='DIR', type=Path,
                    help='where the depth maps are stored, must be a 3D npy file')
15
parser.add_argument('--evaluation_list_path', metavar='PATH', type=Path,
16
17
18
19
20
21
                    help='File with list of images to test for depth evaluation')
parser.add_argument('--flight_path_vector_list', metavar='PATH', type=Path,
                    help='File with list of speed vectors, used to compute error wrt direction')
parser.add_argument('--scale-invariant', action='store_true',
                    help='If selected, will rescale depth map with ratio of medians')

22
coords = None
23
24


Clément Pinard's avatar
Clément Pinard committed
25
def get_values(gt_depth, estim_depth, fpv, scale_invariant=False, mask=None, min_depth=1e-2, max_depth=250):
26
27
    global coords
    if coords is None:
28
        coords = np.stack(np.meshgrid(np.arange(gt_depth.shape[1]), np.arange(gt_depth.shape[0])), axis=-1)
29
    fpv_dist = np.linalg.norm(coords - fpv, axis=-1)
30
31
    estim_depth = np.clip(estim_depth, min_depth, max_depth)
    valid = (gt_depth > min_depth) & (gt_depth < max_depth)
Clément Pinard's avatar
Clément Pinard committed
32
33
34
35
36
37
38
    if mask is not None:
        valid = valid & mask
    if valid.sum() == 0:
        return
    valid_gt, valid_estim = gt_depth[valid], estim_depth[valid]
    if scale_invariant:
        valid_estim = valid_estim * np.median(valid_gt) / np.median(valid_estim)
39
    fpv_dist = fpv_dist[valid]
40
    valid_coords = coords[valid]
Clément Pinard's avatar
Clément Pinard committed
41
    values = np.stack([valid_gt, valid_estim, *valid_coords.T, fpv_dist], axis=-1)
42
43
44
45

    return pd.DataFrame(values, columns=["GT", "estim", "x", "y", "fpv_dist"])


46
def plot_distribution(values, bins, ax, label=None, log_bins=False):
47
48
    bin_dists = bins[1:] - bins[:-1]
    total = sum(bin_dists)
49
    normalized_values = (values / sum(values)) * bin_dists / total
50
    bin_centers = 0.5*(bins[1:] + bins[:-1])
51
52
53
54
55
    if log_bins:
        bin_centers = np.exp(bin_centers)
    ax.plot(bin_centers, normalized_values, label=label)
    if log_bins:
        ax.set_xscale('log')
56
57


58
59
60
61
62
63
64
65
66
67
68
69
70
71
def group_quantiles(df, to_group, columns, quantiles=[0.25, 0.5, 0.75]):
    if isinstance(columns, str):
        columns = [columns]
    grouped_df = df.groupby(by=np.round(df[to_group]))
    return grouped_df[columns].quantile(quantiles).unstack()


def error_map(error_per_px):
    x, y = np.stack(error_per_px.index.values, axis=-1).astype(int)
    error_map = np.full((int(x.max() + 1), int(y.max() + 1)), np.NaN)
    error_map[x, y] = error_per_px.values
    return error_map


72
73
def main():
    args = parser.parse_args()
74
    n_bins = 4
75
    with open(args.evaluation_list_path, 'r') as f:
76
        depth_paths = [line[:-1] for line in f.readlines()]
77
78
79
80
    fpv_list = np.loadtxt(args.flight_path_vector_list)
    estimated_depth = np.load(args.est_depth, allow_pickle=True)
    values_df = []
    assert(len(depth_paths) == len(estimated_depth))
Clément Pinard's avatar
Clément Pinard committed
81
    for filepath, fpv in tqdm(zip(depth_paths, fpv_list), total=len(fpv_list)):
82
83
        GT = np.load(args.dataset_root/filepath + '.npy')
        new_values = get_values(GT, estimated_depth[filepath], fpv)
Clément Pinard's avatar
Clément Pinard committed
84
85
        if new_values is not None:
            values_df.append(new_values)
86
87

    values_df = pd.concat(values_df)
88
89
    values_df["log_GT"] = np.log(values_df["GT"])
    values_df["log_estim"] = np.log(values_df["estim"])
90
91
    values_df["diff"] = values_df["estim"] - values_df["GT"]
    values_df["absdiff"] = values_df["diff"].abs()
92
    values_df["reldiff"] = values_df["diff"] / values_df["GT"]
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
    values_df["logdiff"] = values_df["log_estim"] - values_df["log_GT"]
    values_df["abslogdiff"] = values_df["logdiff"].abs()

    error_names = ["AbsDiff", "StdDiff", "AbsRel", "StdRel", "AbsLog", "StdLog", "a1", "a2", "a3"]
    errors = [values_df["diff"].mean(),
              np.sqrt(np.power(values_df["diff"], 2).mean()),
              values_df["reldiff"].mean(),
              np.sqrt(np.power(values_df["reldiff"], 2).mean()),
              values_df["logdiff"].abs().mean(),
              np.sqrt(np.power(values_df["logdiff"], 2).mean()),
              sum(values_df["logdiff"] < np.log(1.25)) / len(values_df),
              sum(values_df["logdiff"] < 2 * np.log(1.25)) / len(values_df),
              sum(values_df["logdiff"] < 3 * np.log(1.25)) / len(values_df)]
    print("Results for usual metrics")
    print("{:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}, {:>10}".format(*error_names))
    print("{:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".format(*errors))
109
110
111
112
113
114
115
116
117
118
119

    plot = True
    if plot:

        min_gt = values_df["GT"].min()
        max_gt = values_df["GT"].max()

        bins = np.linspace(min_gt, max_gt, n_bins + 1)

        estim_per_GT = {}
        for b1, b2 in zip(bins[:-1], bins[1:]):
120
            per_gt = values_df[(values_df["GT"] > b1) & (values_df["GT"] < b2)]
121
122
            estim_per_GT[(b1+b2)/2] = {"normal": np.histogram(per_gt["diff"], bins=100),
                                       "log_normal": np.histogram(per_gt["logdiff"], bins=100),
123
                                       "bins": [b1, b2]}
124

125
        global_diff = np.histogram(values_df["estim"] - values_df["GT"], bins=100)
126

127
        global_log_diff = np.histogram(values_df["log_estim"] - values_df["log_GT"], bins=100)
128

129
        metric_per_px = values_df.groupby(by=["x", "y"]).mean()
130
        mean_diff_per_px = error_map(metric_per_px["absdiff"])
131
        mean_log_diff_per_px = error_map(metric_per_px["logdiff"])
132

133
134
135
136
        quantiles_per_fpv = group_quantiles(values_df[values_df["fpv_dist"] < 1000],
                                            "fpv_dist",
                                            ["absdiff", "abslogdiff"])
        quantiles_per_gt = group_quantiles(values_df, "GT", ["absdiff", "abslogdiff"])
137

138
139
140
141
142
143
144
145
146
        # metric_per_gt = values_df.groupby(by = np.round(values_df["GT"]))

        fig, axes = plt.subplots(2, 1, sharex=True)
        GT_distrib = np.histogram(values_df["GT"], bins=100)
        plot_distribution(*GT_distrib, axes[0])
        axes[0].set_title("Ground Truth distribution")
        estim_distrib = np.histogram(values_df["estim"], bins=100)
        plot_distribution(*estim_distrib, axes[1])
        axes[1].set_title("depth estimation distribution")
147

148
        fig, axes = plt.subplots(1, 2, sharey=True)
149
        for i, (k, v) in enumerate(estim_per_GT.items()):
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
            plot_distribution(*v["normal"], axes[0], label="$GT \\in [{:.1f},{:.1f}]$".format(*v["bins"]))
            plot_distribution(*v["log_normal"], axes[1], label="$GT \\in [{:.1f},{:.1f}]$".format(*v["bins"]), log_bins=True)
            axes[0].legend()
            axes[1].legend()
            # axes[0, 0].set_title("dstribution of estimation around GT = {:.2f}".format(k))
            # axes[0, 1].set_title("dstribution of log estimation around log GT = {:.2f}".format(np.log(k)))

        fig, axes = plt.subplots(2, 1)
        plot_distribution(*global_diff, axes[0])
        axes[0].set_title("Global difference distribution from GT")
        plot_distribution(*global_log_diff, axes[1], log_bins=True)
        axes[1].set_title("Global log difference distribution from GT")

        plt.tight_layout()
        fig, axes = plt.subplots(2, 1, sharex=True)
165
166
167
        index = quantiles_per_fpv.index
        diff_per_fpv = quantiles_per_fpv["absdiff"]
        logdiff_per_fpv = quantiles_per_fpv["abslogdiff"]
Clément Pinard's avatar
Clément Pinard committed
168
169
        axes[0].fill_between(index, diff_per_fpv[0.25], diff_per_fpv[0.75], color="cyan", label="25% - 75%")
        axes[0].plot(diff_per_fpv[0.5].index, diff_per_fpv[0.5], label="median")
170
        axes[0].set_title("Error wrt to distance to fpv (in px)")
Clément Pinard's avatar
Clément Pinard committed
171
172
        axes[1].fill_between(index, logdiff_per_fpv[0.25], logdiff_per_fpv[0.75], color="cyan", label="25% - 75%")
        axes[1].plot(logdiff_per_fpv[0.5], label="median")
173
        axes[1].set_title("Log error wrt to distance to fpv (in px)")
Clément Pinard's avatar
Clément Pinard committed
174
        axes[1].set_xlabel("Distance to flight path vector (in px)")
175
176
177
178
179
180

        plt.tight_layout()
        fig, axes = plt.subplots(2, 1, sharex=True)
        index = quantiles_per_gt.index
        diff_per_gt = quantiles_per_gt["absdiff"]
        logdiff_per_gt = quantiles_per_gt["abslogdiff"]
Clément Pinard's avatar
Clément Pinard committed
181
182
        axes[0].fill_between(index, diff_per_gt[0.25], diff_per_gt[0.75], color="cyan", label="25% - 75%")
        axes[0].plot(diff_per_gt[0.5], label="median")
183
        axes[0].set_title("Error wrt to distance to groundtruth depth")
Clément Pinard's avatar
Clément Pinard committed
184
185
        axes[1].fill_between(index, logdiff_per_gt[0.25], logdiff_per_gt[0.75], color="cyan", label="25% - 75%")
        axes[1].plot(logdiff_per_gt[0.5], label="median")
186
        axes[1].set_title("Log error wrt to groundtruth depth")
Clément Pinard's avatar
Clément Pinard committed
187
        axes[1].set_xlabel("Groundtruth depth (in meters)")
188
189
190

        plt.tight_layout()
        fig, axes = plt.subplots(2, 1)
Clément Pinard's avatar
Clément Pinard committed
191
        pl = axes[0].imshow(mean_diff_per_px.T)
192
        axes[0].set_title("Mean error for each pixel")
Clément Pinard's avatar
Clément Pinard committed
193
194
195
196
197
198
        divider = make_axes_locatable(axes[0])
        cax = divider.append_axes("right", size="5%", pad=0.05)
        cbar = fig.colorbar(pl, cax=cax)
        cbar.ax.tick_params(axis='y', direction='in')
        cbar.set_label('# Mean abs diff', rotation=270)
        pl = axes[1].imshow(mean_log_diff_per_px.T)
199
        axes[1].set_title("Mean Log error for each pixel")
Clément Pinard's avatar
Clément Pinard committed
200
201
202
203
204
        divider = make_axes_locatable(axes[1])
        cax = divider.append_axes("right", size="5%", pad=0.05)
        cbar = fig.colorbar(pl, cax=cax)
        cbar.ax.tick_params(axis='y', direction='in')
        cbar.set_label('# Mean abs log diff', rotation=270)
205
        plt.tight_layout()
206
207
        plt.show()

208
209
210

if __name__ == '__main__':
    main()