From 0f322b208a5c5cbbbcfff911ede92442d479ecb3 Mon Sep 17 00:00:00 2001 From: Christian Forster Date: Fri, 16 May 2014 19:25:51 +0200 Subject: [PATCH] evaluation scripts --- svo_analysis/scripts/comparison.py | 26 +++++++- svo_analysis/scripts/evaluate.py | 60 +++++++++++-------- svo_analysis/src/svo_analysis/analyse_logs.py | 14 +++-- .../src/svo_analysis/analyse_trajectory.py | 2 +- 4 files changed, 68 insertions(+), 34 deletions(-) diff --git a/svo_analysis/scripts/comparison.py b/svo_analysis/scripts/comparison.py index 8f99f01..f2ac8a7 100755 --- a/svo_analysis/scripts/comparison.py +++ b/svo_analysis/scripts/comparison.py @@ -8,6 +8,7 @@ import numpy as np import matplotlib.pyplot as plt import yaml import argparse +import svo_analysis.analyse_depth as analyse_depth from matplotlib import rc # tell matplotlib to use latex font @@ -43,6 +44,7 @@ def compare_results(comp_params, results_dir, comparison_dir): print('run comparison: '+comp_params['comparison_name']) line_styles = ['-','--',':'] + line_colors = ['b','g','r','m','c'] # ------------------------------------------------------------------------- # plot trajectory: @@ -57,10 +59,9 @@ def compare_results(comp_params, results_dir, comparison_dir): base_plot, = ax.plot(data[:,1], data[:,2], label=exp_set['label'], linestyle=line_styles[np.mod(i, len(line_styles))]) else: ax.plot(data[:,1], data[:,2], color=base_plot.get_color(), linestyle= line_styles[np.mod(i, len(line_styles))]) - ax.legend(loc='upper left', ncol=2) + ax.legend(loc='upper left', ncol=3) save_figure(fig, 'trajectory', comparison_dir) - # ------------------------------------------------------------------------- # plot translation error: fig = plt.figure(figsize=(6,2)) @@ -93,8 +94,27 @@ def compare_results(comp_params, results_dir, comparison_dir): else: ax.plot(distances, e*1000, color=base_plot.get_color(), linestyle= line_styles[np.mod(i, len(line_styles))]) - ax.legend(loc='upper left', ncol=2) + ax.legend(loc='upper left', ncol=3) save_figure(fig, 'translation_error', comparison_dir) + + # ------------------------------------------------------------------------- + # plot depth estimation error: + fig = plt.figure(figsize=(6,5)) + ax = fig.add_subplot(111, xlabel='Error [m]', ylabel='Precision [\%]') + for k, exp_set in enumerate(comp_params['experiment_sets']): + print('plot depth error for experiment: ' + exp_set['label']) + + for i, exp in enumerate(exp_set['experiments']): + + D = np.loadtxt(os.path.join(results_dir, exp, 'depth_error.txt'), delimiter=' ') + D = D[np.logical_and(D[:,0] > 1, D[:,0] < D[-1,0]/4),:] + errors = np.abs(D[:,1]) + if i == 0: + base_plot = analyse_depth.precision_plot(ax, errors, 0.05, line_colors[k], line_styles[np.mod(i, len(line_styles))]) + else: + base_plot = analyse_depth.precision_plot(ax, errors, 0.05, base_plot.get_color(), line_styles[np.mod(i, len(line_styles))]) + + save_figure(fig, 'depth_error', comparison_dir) if __name__ == '__main__': diff --git a/svo_analysis/scripts/evaluate.py b/svo_analysis/scripts/evaluate.py index eb9fe60..726544c 100755 --- a/svo_analysis/scripts/evaluate.py +++ b/svo_analysis/scripts/evaluate.py @@ -10,41 +10,49 @@ import csv import numpy as np import rospkg import argparse +import yaml import svo_analysis.analyse_logs as analyse_logs import svo_analysis.analyse_timing as analyse_timing import svo_analysis.analyse_trajectory as analyse_trajectory +import svo_analysis.analyse_depth as analyse_depth def evaluate_dataset(trace_dir): - # read tracefile - data = csv.reader(open(os.path.join(trace_dir, 'trace.csv'))) - fields = data.next() - D = dict() - for field in fields: - D[field] = list() + # read tracefile + data = csv.reader(open(os.path.join(trace_dir, 'trace.csv'))) + fields = data.next() + D = dict() + for field in fields: + D[field] = list() - # fill dictionary with column values - for row in data: - for (field, value) in zip(fields, row): - D[field].append(float(value)) + # fill dictionary with column values + for row in data: + for (field, value) in zip(fields, row): + D[field].append(float(value)) - # change dictionary values from list to numpy array for easier manipulation - for field, value in D.items(): - D[field] = np.array(D[field]) + # change dictionary values from list to numpy array for easier manipulation + for field, value in D.items(): + D[field] = np.array(D[field]) + + # generate plots + analyse_logs.analyse_logs(D, trace_dir) + analyse_timing.analyse_timing(D, trace_dir) + analyse_trajectory.analyse_trajectory(trace_dir) + + param = yaml.load(open(os.path.join(trace_dir, 'dataset_params.yaml'), 'r')) + if param['dataset_is_blender']: + analyse_depth.analyse_depth(trace_dir) + analyse_depth.analyse_depth_over_time(trace_dir) - # generate plots - analyse_logs.analyse_logs(D, trace_dir) - analyse_timing.analyse_timing(D, trace_dir) - analyse_trajectory.analyse_trajectory(trace_dir) if __name__=="__main__": - parser = argparse.ArgumentParser(description=''' - Evaluates tracefiles of SVO and generates the plots. - ''') - parser.add_argument('experiment_name', help='directory name of the tracefiles') - args = parser.parse_args() + parser = argparse.ArgumentParser(description=''' + Evaluates tracefiles of SVO and generates the plots. + ''') + parser.add_argument('experiment_name', help='directory name of the tracefiles') + args = parser.parse_args() - trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'), - 'results', - args.experiment_name) - evaluate_dataset(trace_dir) \ No newline at end of file + trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'), + 'results', + args.experiment_name) + evaluate_dataset(trace_dir) \ No newline at end of file diff --git a/svo_analysis/src/svo_analysis/analyse_logs.py b/svo_analysis/src/svo_analysis/analyse_logs.py index 4e40bfe..7dc2acc 100755 --- a/svo_analysis/src/svo_analysis/analyse_logs.py +++ b/svo_analysis/src/svo_analysis/analyse_logs.py @@ -1,6 +1,7 @@ #!/usr/bin/python import os +import yaml import numpy as np import matplotlib.pyplot as plt @@ -10,10 +11,7 @@ def analyse_logs(D, trace_dir): is_kf = np.argwhere( (D['dropout'] == 1) & (D['repr_n_mps'] >= 0)) is_frame = np.argwhere(D['repr_n_mps'] >= 0) is_nokf = np.argwhere( (D['dropout'] == 0) & (D['repr_n_mps'] >= 0)) - n_frames = len(is_frame) - n_kfs = len(is_kf) - n_nokf = len(is_nokf) - + # set initial time to zero D['timestamp'] = D['timestamp'] - D['timestamp'][0] @@ -65,4 +63,12 @@ def analyse_logs(D, trace_dir): ax.plot(D['timestamp'][is_frame], D['sfba_thresh'][is_frame], 'r-', label='Threshold') fig.tight_layout() fig.savefig(os.path.join(trace_dir,'optimization_thresh.pdf'), bbox_inches="tight") + + # ---------------------------------------------------------------------------- + # write other statistics to file + stat = {'num_frames': len(is_frame), + 'num_kfs': len(is_kf), + 'reproj_error_avg_improvement': float(init_error_avg - opt1_avg)} + with open(os.path.join(trace_dir,'dataset_stats.yaml'),'w') as outfile: + outfile.write(yaml.dump(stat, default_flow_style=False)) diff --git a/svo_analysis/src/svo_analysis/analyse_trajectory.py b/svo_analysis/src/svo_analysis/analyse_trajectory.py index 4b11363..15823d0 100755 --- a/svo_analysis/src/svo_analysis/analyse_trajectory.py +++ b/svo_analysis/src/svo_analysis/analyse_trajectory.py @@ -100,7 +100,7 @@ def analyse_trajectory(results_dir, n_align_frames = 200, use_hand_eye_calib = T ax = fig.add_subplot(111, title='trajectory', aspect='equal', xlabel='x [m]', ylabel='y [m]') ax.plot(p_es_aligned[:,0], p_es_aligned[:,1], 'b-', label='estimate') ax.plot(p_gt[:,0], p_gt[:,1], 'r-', label='groundtruth') - #ax.plot(p_es_aligned[0:n_align_frames,0], p_es_aligned[0:n_align_frames,1], 'g-', linewidth=2, label='aligned') + ax.plot(p_es_aligned[0:n_align_frames,0], p_es_aligned[0:n_align_frames,1], 'g-', linewidth=2, label='aligned') ax.legend() fig.tight_layout() fig.savefig(results_dir+'/trajectory.pdf')