public/vio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactored benchmarking

Browse Source
This commit is contained in:
Christian Forster
2014-03-27 13:46:15 +01:00
parent ce0e6543aa
commit 674cf55ad3
4 changed files with 100 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
svo_analysis/scripts/benchmark.py
+2 -3
View File
@@ -10,9 +10,9 @@ import yaml
import rospkg import rospkg
import argparse import argparse
import time import time
import svo_analysis.analyse_trajectory as analyse_trajectory
import vikit_py.cpu_info as cpu_info import vikit_py.cpu_info as cpu_info
import vikit_py.ros_node as ros_node import vikit_py.ros_node as ros_node
import evaluate
if __name__=="__main__": if __name__=="__main__":
@@ -67,8 +67,7 @@ if __name__=="__main__":
node.run(params) node.run(params)
# TODO: check if it is a synthetic dataset # TODO: check if it is a synthetic dataset
# plot trajectory error evaluate.evaluate_dataset(params['trace_dir'])
analyse_trajectory.analyse_trajectory(params['trace_dir'])
svo_analysis/scripts/evaluate.py
+22 -19
View File
@@ -5,26 +5,19 @@ Created on Sat Aug 10 18:21:47 2013
@author: Christian Forster @author: Christian Forster
""" """
import argparse import os
import csv import csv
import numpy as np import numpy as np
from matplotlib import rc import rospkg
import argparse
import svo_analysis.analyse_logs as analyse_logs import svo_analysis.analyse_logs as analyse_logs
import svo_analysis.analyse_timing as analyse_timing import svo_analysis.analyse_timing as analyse_timing
import svo_analysis.analyse_trajectory as analyse_trajectory
if __name__=="__main__": def evaluate_dataset(trace_dir):
# parse command line # read tracefile
parser = argparse.ArgumentParser() data = csv.reader(open(os.path.join(trace_dir, 'trace.csv')))
parser.add_argument('--trace_name', help='svo tracefile with timings and logs', default='20131121_0930_px7')
args = parser.parse_args()
# config
results_dir = 'results/'+args.trace_name
trace_file = results_dir+'/'+args.trace_name+'.csv'
# tracefile
data = csv.reader(open(trace_file))
fields = data.next() fields = data.next()
D = dict() D = dict()
for field in fields: for field in fields:
@@ -39,9 +32,19 @@ if __name__=="__main__":
for field, value in D.items(): for field, value in D.items():
D[field] = np.array(D[field]) D[field] = np.array(D[field])
# tell matplotlib to use latex font # generate plots
rc('font',**{'family':'serif','serif':['Cardo']}) analyse_logs.analyse_logs(D, trace_dir)
rc('text', usetex=True) analyse_timing.analyse_timing(D, trace_dir)
analyse_trajectory.analyse_trajectory(trace_dir)
analyse_logs.analyse_logs(D, results_dir+'/'+args.trace_name) if __name__=="__main__":
analyse_timing.analyse_timing(D, results_dir+'/'+args.trace_name) parser = argparse.ArgumentParser(description='''
Evaluates tracefiles of SVO and generates the plots.
''')
parser.add_argument('experiment_name', help='directory name of the tracefiles')
args = parser.parse_args()
trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'),
'results',
args.experiment_name)
evaluate_dataset(trace_dir)
svo_analysis/src/svo_analysis/analyse_logs.py
+36 -100
View File
@@ -1,16 +1,10 @@
#!/usr/bin/python #!/usr/bin/python
import argparse import os
import csv
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from matplotlib import rc
# tell matplotlib to use latex font
rc('font',**{'family':'serif','serif':['Cardo']})
rc('text', usetex=True)
def analyse_logs(D, trace_name): def analyse_logs(D, trace_dir):
# identify measurements which result from normal frames and which from keyframes # identify measurements which result from normal frames and which from keyframes
is_kf = np.argwhere( (D['dropout'] == 1) & (D['repr_n_mps'] >= 0)) is_kf = np.argwhere( (D['dropout'] == 1) & (D['repr_n_mps'] >= 0))
@@ -19,114 +13,56 @@ def analyse_logs(D, trace_name):
n_frames = len(is_frame) n_frames = len(is_frame)
n_kfs = len(is_kf) n_kfs = len(is_kf)
n_nokf = len(is_nokf) n_nokf = len(is_nokf)
fps = 25.0
# set initial time to zero
D['timestamp'] = D['timestamp'] - D['timestamp'][0]
# ----------------------------------------------------------------------------
# plot number of reprojected points # plot number of reprojected points
mean_n_reproj_points = np.mean(D['repr_n_mps'][is_frame]); mean_n_reproj_points = np.mean(D['repr_n_mps'][is_frame]);
mean_n_reproj_matches = np.mean(D['repr_n_new_references'][is_frame]); mean_n_reproj_matches = np.mean(D['repr_n_new_references'][is_frame]);
mean_n_edges_final = np.mean(D['sfba_n_edges_final'][is_frame]);
plt.figure() fig = plt.figure(figsize=(8,3))
plt.plot(np.arange(n_frames), D['repr_n_mps'][is_frame], 'r-', label='reprojected points, avg = %(i).2f'%{'i':mean_n_reproj_points}) ax = fig.add_subplot(111, xlabel='time [s]')
plt.plot(np.arange(n_frames), D['repr_n_new_references'][is_frame], 'b-', label='reprojected matches, avg = %(i).2f'%{'i':mean_n_reproj_matches}) ax.plot(D['timestamp'][is_frame], D['repr_n_mps'][is_frame], 'r-',
plt.legend() label='Reprojected Points, avg = %.2f'%mean_n_reproj_points)
plt.tight_layout() ax.plot(D['timestamp'][is_frame], D['repr_n_new_references'][is_frame], 'b-',
plt.title('Reprojected points asdf') label='Feature Matches, avg = %.2f'%mean_n_reproj_matches)
plt.savefig(trace_name+'_reprojection.pdf', bbox_inches="tight") ax.plot(D['timestamp'][is_frame], D['sfba_n_edges_final'][is_frame], 'g-',
label='Points after Optimization, avg = %.2f'%mean_n_edges_final)
# plot number of triangulations ax.set_ylim(bottom=0)
# plt.figure(3) ax.legend(loc='lower right')
# plt.plot(np.arange(n_kfs), D['triang_n_points'][is_kf], 'g-', label='triangulated points, avg %0.2f'%np.mean(D['triang_n_points'][is_kf]))
# plt.plot(np.arange(n_kfs), D['triang_n_refs'][is_kf], 'b-', label='new references, avg %0.2f'%np.mean(D['triang_n_refs'][is_kf]))
# plt.plot(np.arange(n_kfs), D['triang_n_fails'][is_kf], 'r-', label='failures, avg %0.2f'%np.mean(D['triang_n_fails'][is_kf]))
# plt.plot(np.arange(n_kfs), D['triang_n_add_refs'][is_kf], 'm-', label='additional refs, avg %0.2f'%np.mean(D['triang_n_add_refs'][is_kf]))
# plt.legend()
# plt.title('Triangulation')
# plt.savefig(trace_name+'_img_triangulation.png', bbox_inches="tight")
# plot number of klt tracked and matched points
fig = plt.figure(figsize=(8,2))
ax = fig.add_subplot(111, xlabel='time [s]', ylabel='no. features')
ax.plot(np.arange(n_frames)/fps, D['repr_n_new_references'][is_frame], 'b-', label='number of successful matches')
fig.tight_layout() fig.tight_layout()
fig.savefig(trace_name+'number_of_features.pdf', bbox_inches="tight") fig.savefig(os.path.join(trace_dir,'num_reprojected.pdf'), bbox_inches="tight")
# ----------------------------------------------------------------------------
# plot median error before and after pose-optimzation and bundle adjustment # plot median error before and after pose-optimzation and bundle adjustment
init_error_avg = np.mean(D['sfba_error_init'][is_frame]) init_error_avg = np.mean(D['sfba_error_init'][is_frame])
opt1_avg = np.mean(D['sfba_error_final'][is_frame]) opt1_avg = np.mean(D['sfba_error_final'][is_frame])
fig = plt.figure(figsize=(8,2)) fig = plt.figure(figsize=(8,2))
ax = fig.add_subplot(111, xlabel='time [s]', ylabel='error [px]') ax = fig.add_subplot(111, xlabel='time [s]', ylabel='error [px]')
t_max = n_frames ax.plot(D['timestamp'][is_frame], D['sfba_error_init'][is_frame], 'r-', label='Initial error')
ax.plot(np.arange(n_frames)[0:t_max]/fps, D['sfba_error_init'][is_frame][0:t_max], 'r-', label='Initial error') ax.plot(D['timestamp'][is_frame], D['sfba_error_final'][is_frame], 'b-', label='Final error')
ax.plot(np.arange(n_frames)[0:t_max]/fps, D['sfba_error_final'][is_frame][0:t_max], 'b-', label='Final error')
ax.legend(ncol=2) ax.legend(ncol=2)
ax.set_xlim([0, t_max/fps+4])
#ax.set_ylim([0, 0.8])
fig.tight_layout() fig.tight_layout()
fig.savefig(trace_name+'reprojection_error.pdf', bbox_inches="tight") fig.savefig(os.path.join(trace_dir,'reprojection_error.pdf'), bbox_inches="tight")
print 'average reprojection error improvement: ' + str(init_error_avg - opt1_avg) print 'average reprojection error improvement: ' + str(init_error_avg - opt1_avg)
# plot number of references # ----------------------------------------------------------------------------
plt.figure() # plot number of candidate points
plt.fill(np.arange(n_frames+1), np.append(D['repr_n_new_references'][is_frame]+D['klt_n_matches'][is_frame],0), 'g-', label='Reprojection') fig = plt.figure(figsize=(8,3))
plt.fill(np.arange(n_frames+1), np.append(D['klt_n_tracked'][is_frame],0), 'b-', label='KLT Matches') ax = fig.add_subplot(111, xlabel='time [s]')
plt.plot(np.arange(n_frames), D['sfba_n_edges_final'][is_frame], 'r-', label='References after Optimization', linewidth=2.0) ax.plot(D['timestamp'][is_frame], D['n_candidates'][is_frame], 'r-', label='Candidate Points')
plt.legend() fig.tight_layout()
plt.tight_layout() fig.savefig(os.path.join(trace_dir,'candidate_points.pdf'), bbox_inches="tight")
plt.title('Pose Optimization')
plt.savefig(trace_name+'_img_n_references.pdf', bbox_inches="tight")
# plot number of seeds # ----------------------------------------------------------------------------
plt.figure() # plot number of candidate points
has_seed = np.argwhere(D['seeds_n_tot'] >= 0) fig = plt.figure(figsize=(8,2))
plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_tot'][has_seed],0), label='nSeeds', edgecolor='none') ax = fig.add_subplot(111, xlabel='time [s]', ylabel='px')
plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_updates'][has_seed]+D['seeds_n_failed_matches'][has_seed],0), label='nFailedMatches', edgecolor='none') ax.plot(D['timestamp'][is_frame], D['sfba_thresh'][is_frame], 'r-', label='Threshold')
plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_updates'][has_seed],0), label='nUpdates', edgecolor='none') fig.tight_layout()
plt.legend() fig.savefig(os.path.join(trace_dir,'optimization_thresh.pdf'), bbox_inches="tight")
plt.tight_layout()
plt.title('Seeds')
plt.savefig(trace_name+'_seeds.pdf', bbox_inches="tight")
plt.figure()
plt.plot(D['n_candidates'])
plt.title('Candidates')
plt.show()
# plt.figure()
# plt.plot())
if __name__=="__main__":
dataset='20131003_1937_nslam_i7_asl2_fast'
n_align_frames = 10
save = True
results_dir = '../results/'+dataset
logfile = results_dir+'/'+dataset+'_0.csv'
# parse command line
parser = argparse.ArgumentParser(description='''
This script computes the absolute trajectory error from the ground truth trajectory and the estimated trajectory.
''')
parser.add_argument('--trace_file', help='nanoslam tracefile with timings and logs', default=logfile)
parser.add_argument('--trace_name', help='name of the tracefile, used for saving figures etc.', default=results_dir+'/')
args = parser.parse_args()
# tracefile
data = csv.reader(open(args.trace_file))
fields = data.next()
D = dict()
for field in fields:
D[field] = list()
# fill dictionary with column values
for row in data:
for (field, value) in zip(fields, row):
D[field].append(float(value))
# change dictionary values from list to numpy array for easier manipulation
for field, value in D.items():
D[field] = np.array(D[field])
analyse_logs(D, args.trace_name)
svo_analysis/src/svo_analysis/analyse_timing.py
+40 -69
View File
@@ -1,97 +1,68 @@
#!/usr/bin/python #!/usr/bin/python
import argparse import os
import csv
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from matplotlib import rc
def analyse_timing(D, trace_name): def analyse_timing(D, trace_dir):
# identify measurements which result from normal frames and which from keyframes # identify measurements which result from normal frames and which from keyframes
is_frame = np.argwhere(D['repr_n_mps'] >= 0) is_frame = np.argwhere(D['repr_n_mps'] >= 0)
n_frames = len(is_frame) n_frames = len(is_frame)
# set initial time to zero
D['timestamp'] = D['timestamp'] - D['timestamp'][0]
# ----------------------------------------------------------------------------
# plot total time for frame processing # plot total time for frame processing
avg_time = np.mean(D['t_tot_time'][is_frame])*1000; avg_time = np.mean(D['tot_time'][is_frame])*1000;
fig = plt.figure(figsize=(8, 3)) fig = plt.figure(figsize=(8, 3))
ax = fig.add_subplot(111, ylabel='processing time [ms]', xlabel='time [s]') ax = fig.add_subplot(111, ylabel='processing time [ms]', xlabel='time [s]')
ax.plot(np.arange(n_frames), D['t_tot_time'][is_frame]*1000, 'g-', label='total time [ms]') ax.plot(D['timestamp'][is_frame], D['tot_time'][is_frame]*1000, 'g-', label='total time [ms]')
ax.plot(np.arange(n_frames), np.ones(n_frames)*avg_time, 'b--', label=str('%(time).1fms mean time' % {'time': avg_time})) ax.plot(D['timestamp'][is_frame], np.ones(n_frames)*avg_time, 'b--', label=str('%(time).1fms mean time' % {'time': avg_time}))
ax.legend() ax.legend()
fig.tight_layout() fig.tight_layout()
fig.savefig(trace_name+'_img_tot_processing_time.pdf', bbox_inches="tight") fig.savefig(os.path.join(trace_dir,'timing.pdf'), bbox_inches="tight")
# ----------------------------------------------------------------------------
# plot boxplot
fig = plt.figure(figsize=(6,2)) fig = plt.figure(figsize=(6,2))
ax = fig.add_subplot(111, xlabel='Processing time [ms]') ax = fig.add_subplot(111, xlabel='Processing time [ms]')
ax.boxplot([ ax.boxplot([
D['t_tot_time'][is_frame]*1000, D['tot_time'][is_frame]*1000,
# D['t_local_ba'][is_kf]*1000, # D['t_local_ba'][is_kf]*1000,
D['t_pose_optimizer'][is_frame]*1000 + D['t_point_optimizer'][is_frame]*1000, D['pose_optimizer'][is_frame]*1000 + D['point_optimizer'][is_frame]*1000,
D['t_reproject'][is_frame]*1000, D['reproject'][is_frame]*1000,
D['t_sparse_img_align'][is_frame]*1000, D['sparse_img_align'][is_frame]*1000,
D['t_pyramid_creation'][is_frame]*1000 D['pyramid_creation'][is_frame]*1000
], 0,'', vert=0) # ], 0,'', vert=0)
boxplot_labels = [ boxplot_labels = [
r'\textbf{Total Motion Estimation: %2.2fms}' % np.median(D['t_tot_time'][is_frame]*1000), r'\textbf{Total Motion Estimation: %2.2fms}' % np.median(D['tot_time'][is_frame]*1000),
# 'Local BA (KF only): %.2fms ' % np.median(D['t_local_ba'][is_kf]*1000), # 'Local BA (KF only): %.2fms ' % np.median(D['local_ba'][is_kf]*1000),
'Refinement: %2.2fms' % np.median(D['t_pose_optimizer'][is_frame]*1000 + D['t_point_optimizer'][is_frame]*1000), 'Refinement: %2.2fms' % np.median(D['pose_optimizer'][is_frame]*1000 + D['point_optimizer'][is_frame]*1000),
'Feature Alignment: %2.2fms' % np.median(D['t_reproject'][is_frame]*1000), 'Feature Alignment: %2.2fms' % np.median(D['reproject'][is_frame]*1000),
'Sparse Image Alignment: %2.2fms' % np.median(D['t_sparse_img_align'][is_frame]*1000), 'Sparse Image Alignment: %2.2fms' % np.median(D['sparse_img_align'][is_frame]*1000),
'Pyramid Creation: %2.2fms' % np.median(D['t_pyramid_creation'][is_frame]*1000) ] 'Pyramid Creation: %2.2fms' % np.median(D['pyramid_creation'][is_frame]*1000) ]
ax.set_yticks(np.arange(len(boxplot_labels))+1) ax.set_yticks(np.arange(len(boxplot_labels))+1)
ax.set_yticklabels(boxplot_labels) ax.set_yticklabels(boxplot_labels)
fig.tight_layout() fig.tight_layout()
fig.savefig(trace_name+'_timing_boxplot.pdf', bbox_inches="tight") fig.savefig(os.path.join(trace_dir,'timing_boxplot.pdf'), bbox_inches="tight")
fig = plt.figure(figsize=(8, 5)) # ----------------------------------------------------------------------------
ax = fig.add_subplot(111, ylabel='time [ms]') # plot boxplot reprojection
ax.plot(D['t_reproject']*1000, 'r', label='reproject') fig = plt.figure(figsize=(6,2))
ax.plot(D['t_reproject_find_pts']*1000, 'g', label='reproject find pts') # ok! ax = fig.add_subplot(111, xlabel='Processing time [ms]')
ax.plot(D['t_reproject_best']*1000, 'm', label='reproject best') ax.boxplot([ D['reproject'][is_frame]*1000,
ax.plot(D['t_reproject_candidates']*1000, 'b', label='reproject candidates') D['feature_align'][is_frame]*1000,
ax.legend() D['reproject_candidates'][is_frame]*1000,
D['reproject_kfs'][is_frame]*1000 ], 0, '', vert=0)
boxplot_labels = [r'\textbf{Total Reprojection: %2.2fms}' % np.median(D['reproject'][is_frame]*1000),
'Feature Alignment: %2.2fms' % np.median(D['feature_align'][is_frame]*1000),
'Reproject Candidates: %2.2fms' % np.median(D['reproject_candidates'][is_frame]*1000),
'Reproject Keyframes: %2.2fms' % np.median(D['reproject_kfs'][is_frame]*1000) ]
ax.set_yticks(np.arange(len(boxplot_labels))+1)
ax.set_yticklabels(boxplot_labels)
fig.tight_layout() fig.tight_layout()
fig.savefig(trace_name+'_reprojection.pdf', bbox_inches="tight") fig.savefig(os.path.join(trace_dir,'timing_reprojection.pdf'), bbox_inches="tight")
if __name__=="__main__":
# config
dataset='20131003_1937_nslam_i7_asl2_fast'
results_dir = '../results/'+dataset
tracefile = results_dir + '/'+dataset+'_0.csv'
# parse command line
parser = argparse.ArgumentParser(description='''
This script computes the absolute trajectory error from the ground truth trajectory and the estimated trajectory.
''')
parser.add_argument('--trace_file', help='nanoslam tracefile with timings and logs', default=tracefile)
parser.add_argument('--trace_name', help='name of the tracefile, used for saving figures etc.', default=results_dir)
args = parser.parse_args()
results_dir = args.trace_name
# tracefile
data = csv.reader(open(args.trace_file))
fields = data.next()
D = dict()
for field in fields:
D[field] = list()
# fill dictionary with column values
for row in data:
for (field, value) in zip(fields, row):
D[field].append(float(value))
# change dictionary values from list to numpy array for easier manipulation
for field, value in D.items():
D[field] = np.array(D[field])
# tell matplotlib to use latex font
rc('font',**{'family':'serif','serif':['Cardo']})
rc('text', usetex=True)