From 674cf55ad309eda25fa16f1e13e08b2afdedb471 Mon Sep 17 00:00:00 2001 From: Christian Forster Date: Thu, 27 Mar 2014 13:46:15 +0100 Subject: [PATCH] refactored benchmarking --- svo_analysis/scripts/benchmark.py | 5 +- svo_analysis/scripts/evaluate.py | 41 +++--- svo_analysis/src/svo_analysis/analyse_logs.py | 136 +++++------------- .../src/svo_analysis/analyse_timing.py | 109 ++++++-------- 4 files changed, 100 insertions(+), 191 deletions(-) diff --git a/svo_analysis/scripts/benchmark.py b/svo_analysis/scripts/benchmark.py index 4cebf88..326a4d7 100755 --- a/svo_analysis/scripts/benchmark.py +++ b/svo_analysis/scripts/benchmark.py @@ -10,9 +10,9 @@ import yaml import rospkg import argparse import time -import svo_analysis.analyse_trajectory as analyse_trajectory import vikit_py.cpu_info as cpu_info import vikit_py.ros_node as ros_node +import evaluate if __name__=="__main__": @@ -67,8 +67,7 @@ if __name__=="__main__": node.run(params) # TODO: check if it is a synthetic dataset - # plot trajectory error - analyse_trajectory.analyse_trajectory(params['trace_dir']) + evaluate.evaluate_dataset(params['trace_dir']) diff --git a/svo_analysis/scripts/evaluate.py b/svo_analysis/scripts/evaluate.py index 4cff109..eb9fe60 100755 --- a/svo_analysis/scripts/evaluate.py +++ b/svo_analysis/scripts/evaluate.py @@ -5,26 +5,19 @@ Created on Sat Aug 10 18:21:47 2013 @author: Christian Forster """ -import argparse +import os import csv import numpy as np -from matplotlib import rc +import rospkg +import argparse import svo_analysis.analyse_logs as analyse_logs import svo_analysis.analyse_timing as analyse_timing +import svo_analysis.analyse_trajectory as analyse_trajectory -if __name__=="__main__": +def evaluate_dataset(trace_dir): - # parse command line - parser = argparse.ArgumentParser() - parser.add_argument('--trace_name', help='svo tracefile with timings and logs', default='20131121_0930_px7') - args = parser.parse_args() - - # config - results_dir = 'results/'+args.trace_name - trace_file = results_dir+'/'+args.trace_name+'.csv' - - # tracefile - data = csv.reader(open(trace_file)) + # read tracefile + data = csv.reader(open(os.path.join(trace_dir, 'trace.csv'))) fields = data.next() D = dict() for field in fields: @@ -39,9 +32,19 @@ if __name__=="__main__": for field, value in D.items(): D[field] = np.array(D[field]) - # tell matplotlib to use latex font - rc('font',**{'family':'serif','serif':['Cardo']}) - rc('text', usetex=True) + # generate plots + analyse_logs.analyse_logs(D, trace_dir) + analyse_timing.analyse_timing(D, trace_dir) + analyse_trajectory.analyse_trajectory(trace_dir) - analyse_logs.analyse_logs(D, results_dir+'/'+args.trace_name) - analyse_timing.analyse_timing(D, results_dir+'/'+args.trace_name) \ No newline at end of file +if __name__=="__main__": + parser = argparse.ArgumentParser(description=''' + Evaluates tracefiles of SVO and generates the plots. + ''') + parser.add_argument('experiment_name', help='directory name of the tracefiles') + args = parser.parse_args() + + trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'), + 'results', + args.experiment_name) + evaluate_dataset(trace_dir) \ No newline at end of file diff --git a/svo_analysis/src/svo_analysis/analyse_logs.py b/svo_analysis/src/svo_analysis/analyse_logs.py index 2f368ec..4e40bfe 100755 --- a/svo_analysis/src/svo_analysis/analyse_logs.py +++ b/svo_analysis/src/svo_analysis/analyse_logs.py @@ -1,16 +1,10 @@ #!/usr/bin/python -import argparse -import csv +import os import numpy as np import matplotlib.pyplot as plt -from matplotlib import rc - -# tell matplotlib to use latex font -rc('font',**{'family':'serif','serif':['Cardo']}) -rc('text', usetex=True) -def analyse_logs(D, trace_name): +def analyse_logs(D, trace_dir): # identify measurements which result from normal frames and which from keyframes is_kf = np.argwhere( (D['dropout'] == 1) & (D['repr_n_mps'] >= 0)) @@ -19,114 +13,56 @@ def analyse_logs(D, trace_name): n_frames = len(is_frame) n_kfs = len(is_kf) n_nokf = len(is_nokf) - fps = 25.0 + # set initial time to zero + D['timestamp'] = D['timestamp'] - D['timestamp'][0] + + # ---------------------------------------------------------------------------- # plot number of reprojected points mean_n_reproj_points = np.mean(D['repr_n_mps'][is_frame]); mean_n_reproj_matches = np.mean(D['repr_n_new_references'][is_frame]); + mean_n_edges_final = np.mean(D['sfba_n_edges_final'][is_frame]); - plt.figure() - plt.plot(np.arange(n_frames), D['repr_n_mps'][is_frame], 'r-', label='reprojected points, avg = %(i).2f'%{'i':mean_n_reproj_points}) - plt.plot(np.arange(n_frames), D['repr_n_new_references'][is_frame], 'b-', label='reprojected matches, avg = %(i).2f'%{'i':mean_n_reproj_matches}) - plt.legend() - plt.tight_layout() - plt.title('Reprojected points asdf') - plt.savefig(trace_name+'_reprojection.pdf', bbox_inches="tight") - - # plot number of triangulations -# plt.figure(3) -# plt.plot(np.arange(n_kfs), D['triang_n_points'][is_kf], 'g-', label='triangulated points, avg %0.2f'%np.mean(D['triang_n_points'][is_kf])) -# plt.plot(np.arange(n_kfs), D['triang_n_refs'][is_kf], 'b-', label='new references, avg %0.2f'%np.mean(D['triang_n_refs'][is_kf])) -# plt.plot(np.arange(n_kfs), D['triang_n_fails'][is_kf], 'r-', label='failures, avg %0.2f'%np.mean(D['triang_n_fails'][is_kf])) -# plt.plot(np.arange(n_kfs), D['triang_n_add_refs'][is_kf], 'm-', label='additional refs, avg %0.2f'%np.mean(D['triang_n_add_refs'][is_kf])) -# plt.legend() -# plt.title('Triangulation') -# plt.savefig(trace_name+'_img_triangulation.png', bbox_inches="tight") - - # plot number of klt tracked and matched points - fig = plt.figure(figsize=(8,2)) - ax = fig.add_subplot(111, xlabel='time [s]', ylabel='no. features') - ax.plot(np.arange(n_frames)/fps, D['repr_n_new_references'][is_frame], 'b-', label='number of successful matches') + fig = plt.figure(figsize=(8,3)) + ax = fig.add_subplot(111, xlabel='time [s]') + ax.plot(D['timestamp'][is_frame], D['repr_n_mps'][is_frame], 'r-', + label='Reprojected Points, avg = %.2f'%mean_n_reproj_points) + ax.plot(D['timestamp'][is_frame], D['repr_n_new_references'][is_frame], 'b-', + label='Feature Matches, avg = %.2f'%mean_n_reproj_matches) + ax.plot(D['timestamp'][is_frame], D['sfba_n_edges_final'][is_frame], 'g-', + label='Points after Optimization, avg = %.2f'%mean_n_edges_final) + ax.set_ylim(bottom=0) + ax.legend(loc='lower right') fig.tight_layout() - fig.savefig(trace_name+'number_of_features.pdf', bbox_inches="tight") + fig.savefig(os.path.join(trace_dir,'num_reprojected.pdf'), bbox_inches="tight") + # ---------------------------------------------------------------------------- # plot median error before and after pose-optimzation and bundle adjustment init_error_avg = np.mean(D['sfba_error_init'][is_frame]) opt1_avg = np.mean(D['sfba_error_final'][is_frame]) fig = plt.figure(figsize=(8,2)) ax = fig.add_subplot(111, xlabel='time [s]', ylabel='error [px]') - t_max = n_frames - ax.plot(np.arange(n_frames)[0:t_max]/fps, D['sfba_error_init'][is_frame][0:t_max], 'r-', label='Initial error') - ax.plot(np.arange(n_frames)[0:t_max]/fps, D['sfba_error_final'][is_frame][0:t_max], 'b-', label='Final error') + ax.plot(D['timestamp'][is_frame], D['sfba_error_init'][is_frame], 'r-', label='Initial error') + ax.plot(D['timestamp'][is_frame], D['sfba_error_final'][is_frame], 'b-', label='Final error') ax.legend(ncol=2) - ax.set_xlim([0, t_max/fps+4]) - #ax.set_ylim([0, 0.8]) fig.tight_layout() - fig.savefig(trace_name+'reprojection_error.pdf', bbox_inches="tight") + fig.savefig(os.path.join(trace_dir,'reprojection_error.pdf'), bbox_inches="tight") print 'average reprojection error improvement: ' + str(init_error_avg - opt1_avg) - # plot number of references - plt.figure() - plt.fill(np.arange(n_frames+1), np.append(D['repr_n_new_references'][is_frame]+D['klt_n_matches'][is_frame],0), 'g-', label='Reprojection') - plt.fill(np.arange(n_frames+1), np.append(D['klt_n_tracked'][is_frame],0), 'b-', label='KLT Matches') - plt.plot(np.arange(n_frames), D['sfba_n_edges_final'][is_frame], 'r-', label='References after Optimization', linewidth=2.0) - plt.legend() - plt.tight_layout() - plt.title('Pose Optimization') - plt.savefig(trace_name+'_img_n_references.pdf', bbox_inches="tight") + # ---------------------------------------------------------------------------- + # plot number of candidate points + fig = plt.figure(figsize=(8,3)) + ax = fig.add_subplot(111, xlabel='time [s]') + ax.plot(D['timestamp'][is_frame], D['n_candidates'][is_frame], 'r-', label='Candidate Points') + fig.tight_layout() + fig.savefig(os.path.join(trace_dir,'candidate_points.pdf'), bbox_inches="tight") - # plot number of seeds - plt.figure() - has_seed = np.argwhere(D['seeds_n_tot'] >= 0) - plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_tot'][has_seed],0), label='nSeeds', edgecolor='none') - plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_updates'][has_seed]+D['seeds_n_failed_matches'][has_seed],0), label='nFailedMatches', edgecolor='none') - plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_updates'][has_seed],0), label='nUpdates', edgecolor='none') - plt.legend() - plt.tight_layout() - plt.title('Seeds') - plt.savefig(trace_name+'_seeds.pdf', bbox_inches="tight") - - - plt.figure() - plt.plot(D['n_candidates']) - plt.title('Candidates') - plt.show() - -# plt.figure() -# plt.plot()) - -if __name__=="__main__": - - dataset='20131003_1937_nslam_i7_asl2_fast' - n_align_frames = 10 - save = True - results_dir = '../results/'+dataset - logfile = results_dir+'/'+dataset+'_0.csv' - - # parse command line - parser = argparse.ArgumentParser(description=''' - This script computes the absolute trajectory error from the ground truth trajectory and the estimated trajectory. - ''') - parser.add_argument('--trace_file', help='nanoslam tracefile with timings and logs', default=logfile) - parser.add_argument('--trace_name', help='name of the tracefile, used for saving figures etc.', default=results_dir+'/') - args = parser.parse_args() - - # tracefile - data = csv.reader(open(args.trace_file)) - fields = data.next() - D = dict() - for field in fields: - D[field] = list() - - # fill dictionary with column values - for row in data: - for (field, value) in zip(fields, row): - D[field].append(float(value)) - - # change dictionary values from list to numpy array for easier manipulation - for field, value in D.items(): - D[field] = np.array(D[field]) - - analyse_logs(D, args.trace_name) + # ---------------------------------------------------------------------------- + # plot number of candidate points + fig = plt.figure(figsize=(8,2)) + ax = fig.add_subplot(111, xlabel='time [s]', ylabel='px') + ax.plot(D['timestamp'][is_frame], D['sfba_thresh'][is_frame], 'r-', label='Threshold') + fig.tight_layout() + fig.savefig(os.path.join(trace_dir,'optimization_thresh.pdf'), bbox_inches="tight") diff --git a/svo_analysis/src/svo_analysis/analyse_timing.py b/svo_analysis/src/svo_analysis/analyse_timing.py index b0d2199..0dfc1bc 100755 --- a/svo_analysis/src/svo_analysis/analyse_timing.py +++ b/svo_analysis/src/svo_analysis/analyse_timing.py @@ -1,97 +1,68 @@ #!/usr/bin/python -import argparse -import csv +import os import numpy as np import matplotlib.pyplot as plt -from matplotlib import rc -def analyse_timing(D, trace_name): +def analyse_timing(D, trace_dir): # identify measurements which result from normal frames and which from keyframes is_frame = np.argwhere(D['repr_n_mps'] >= 0) n_frames = len(is_frame) + + # set initial time to zero + D['timestamp'] = D['timestamp'] - D['timestamp'][0] + # ---------------------------------------------------------------------------- # plot total time for frame processing - avg_time = np.mean(D['t_tot_time'][is_frame])*1000; - + avg_time = np.mean(D['tot_time'][is_frame])*1000; fig = plt.figure(figsize=(8, 3)) ax = fig.add_subplot(111, ylabel='processing time [ms]', xlabel='time [s]') - ax.plot(np.arange(n_frames), D['t_tot_time'][is_frame]*1000, 'g-', label='total time [ms]') - ax.plot(np.arange(n_frames), np.ones(n_frames)*avg_time, 'b--', label=str('%(time).1fms mean time' % {'time': avg_time})) + ax.plot(D['timestamp'][is_frame], D['tot_time'][is_frame]*1000, 'g-', label='total time [ms]') + ax.plot(D['timestamp'][is_frame], np.ones(n_frames)*avg_time, 'b--', label=str('%(time).1fms mean time' % {'time': avg_time})) ax.legend() fig.tight_layout() - fig.savefig(trace_name+'_img_tot_processing_time.pdf', bbox_inches="tight") + fig.savefig(os.path.join(trace_dir,'timing.pdf'), bbox_inches="tight") + # ---------------------------------------------------------------------------- + # plot boxplot fig = plt.figure(figsize=(6,2)) ax = fig.add_subplot(111, xlabel='Processing time [ms]') ax.boxplot([ - D['t_tot_time'][is_frame]*1000, + D['tot_time'][is_frame]*1000, # D['t_local_ba'][is_kf]*1000, - D['t_pose_optimizer'][is_frame]*1000 + D['t_point_optimizer'][is_frame]*1000, - D['t_reproject'][is_frame]*1000, - D['t_sparse_img_align'][is_frame]*1000, - D['t_pyramid_creation'][is_frame]*1000 - ], 0,'', vert=0) # + D['pose_optimizer'][is_frame]*1000 + D['point_optimizer'][is_frame]*1000, + D['reproject'][is_frame]*1000, + D['sparse_img_align'][is_frame]*1000, + D['pyramid_creation'][is_frame]*1000 + ], 0,'', vert=0) boxplot_labels = [ - r'\textbf{Total Motion Estimation: %2.2fms}' % np.median(D['t_tot_time'][is_frame]*1000), -# 'Local BA (KF only): %.2fms ' % np.median(D['t_local_ba'][is_kf]*1000), - 'Refinement: %2.2fms' % np.median(D['t_pose_optimizer'][is_frame]*1000 + D['t_point_optimizer'][is_frame]*1000), - 'Feature Alignment: %2.2fms' % np.median(D['t_reproject'][is_frame]*1000), - 'Sparse Image Alignment: %2.2fms' % np.median(D['t_sparse_img_align'][is_frame]*1000), - 'Pyramid Creation: %2.2fms' % np.median(D['t_pyramid_creation'][is_frame]*1000) ] + r'\textbf{Total Motion Estimation: %2.2fms}' % np.median(D['tot_time'][is_frame]*1000), +# 'Local BA (KF only): %.2fms ' % np.median(D['local_ba'][is_kf]*1000), + 'Refinement: %2.2fms' % np.median(D['pose_optimizer'][is_frame]*1000 + D['point_optimizer'][is_frame]*1000), + 'Feature Alignment: %2.2fms' % np.median(D['reproject'][is_frame]*1000), + 'Sparse Image Alignment: %2.2fms' % np.median(D['sparse_img_align'][is_frame]*1000), + 'Pyramid Creation: %2.2fms' % np.median(D['pyramid_creation'][is_frame]*1000) ] ax.set_yticks(np.arange(len(boxplot_labels))+1) ax.set_yticklabels(boxplot_labels) fig.tight_layout() - fig.savefig(trace_name+'_timing_boxplot.pdf', bbox_inches="tight") + fig.savefig(os.path.join(trace_dir,'timing_boxplot.pdf'), bbox_inches="tight") - fig = plt.figure(figsize=(8, 5)) - ax = fig.add_subplot(111, ylabel='time [ms]') - ax.plot(D['t_reproject']*1000, 'r', label='reproject') - ax.plot(D['t_reproject_find_pts']*1000, 'g', label='reproject find pts') # ok! - ax.plot(D['t_reproject_best']*1000, 'm', label='reproject best') - ax.plot(D['t_reproject_candidates']*1000, 'b', label='reproject candidates') - ax.legend() + # ---------------------------------------------------------------------------- + # plot boxplot reprojection + fig = plt.figure(figsize=(6,2)) + ax = fig.add_subplot(111, xlabel='Processing time [ms]') + ax.boxplot([ D['reproject'][is_frame]*1000, + D['feature_align'][is_frame]*1000, + D['reproject_candidates'][is_frame]*1000, + D['reproject_kfs'][is_frame]*1000 ], 0, '', vert=0) + boxplot_labels = [r'\textbf{Total Reprojection: %2.2fms}' % np.median(D['reproject'][is_frame]*1000), + 'Feature Alignment: %2.2fms' % np.median(D['feature_align'][is_frame]*1000), + 'Reproject Candidates: %2.2fms' % np.median(D['reproject_candidates'][is_frame]*1000), + 'Reproject Keyframes: %2.2fms' % np.median(D['reproject_kfs'][is_frame]*1000) ] + ax.set_yticks(np.arange(len(boxplot_labels))+1) + ax.set_yticklabels(boxplot_labels) fig.tight_layout() - fig.savefig(trace_name+'_reprojection.pdf', bbox_inches="tight") - - -if __name__=="__main__": - - # config - dataset='20131003_1937_nslam_i7_asl2_fast' - results_dir = '../results/'+dataset - tracefile = results_dir + '/'+dataset+'_0.csv' - - # parse command line - parser = argparse.ArgumentParser(description=''' - This script computes the absolute trajectory error from the ground truth trajectory and the estimated trajectory. - ''') - parser.add_argument('--trace_file', help='nanoslam tracefile with timings and logs', default=tracefile) - parser.add_argument('--trace_name', help='name of the tracefile, used for saving figures etc.', default=results_dir) - args = parser.parse_args() - results_dir = args.trace_name - - # tracefile - data = csv.reader(open(args.trace_file)) - fields = data.next() - D = dict() - for field in fields: - D[field] = list() - - # fill dictionary with column values - for row in data: - for (field, value) in zip(fields, row): - D[field].append(float(value)) - - # change dictionary values from list to numpy array for easier manipulation - for field, value in D.items(): - D[field] = np.array(D[field]) - - # tell matplotlib to use latex font - rc('font',**{'family':'serif','serif':['Cardo']}) - rc('text', usetex=True) - - \ No newline at end of file + fig.savefig(os.path.join(trace_dir,'timing_reprojection.pdf'), bbox_inches="tight") \ No newline at end of file