public/vio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactored benchmarking

Browse Source
This commit is contained in:
Christian Forster
2014-03-27 13:46:15 +01:00
parent ce0e6543aa
commit 674cf55ad3
4 changed files with 100 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
svo_analysis/scripts/benchmark.py
+2 -3
View File
@@ -10,9 +10,9 @@ import yaml
import rospkg
import argparse
import time
import svo_analysis.analyse_trajectory as analyse_trajectory
import vikit_py.cpu_info as cpu_info
import vikit_py.ros_node as ros_node
import evaluate
if __name__=="__main__":
@@ -67,8 +67,7 @@ if __name__=="__main__":
node.run(params)
# TODO: check if it is a synthetic dataset
# plot trajectory error
analyse_trajectory.analyse_trajectory(params['trace_dir'])
evaluate.evaluate_dataset(params['trace_dir'])
svo_analysis/scripts/evaluate.py
+22 -19
View File
@@ -5,26 +5,19 @@ Created on Sat Aug 10 18:21:47 2013
@author: Christian Forster
"""
import argparse
import os
import csv
import numpy as np
from matplotlib import rc
import rospkg
import argparse
import svo_analysis.analyse_logs as analyse_logs
import svo_analysis.analyse_timing as analyse_timing
import svo_analysis.analyse_trajectory as analyse_trajectory
if __name__=="__main__":
def evaluate_dataset(trace_dir):
# parse command line
parser = argparse.ArgumentParser()
parser.add_argument('--trace_name', help='svo tracefile with timings and logs', default='20131121_0930_px7')
args = parser.parse_args()
# config
results_dir = 'results/'+args.trace_name
trace_file = results_dir+'/'+args.trace_name+'.csv'
# tracefile
data = csv.reader(open(trace_file))
# read tracefile
data = csv.reader(open(os.path.join(trace_dir, 'trace.csv')))
fields = data.next()
D = dict()
for field in fields:
@@ -39,9 +32,19 @@ if __name__=="__main__":
for field, value in D.items():
D[field] = np.array(D[field])
# tell matplotlib to use latex font
rc('font',**{'family':'serif','serif':['Cardo']})
rc('text', usetex=True)
# generate plots
analyse_logs.analyse_logs(D, trace_dir)
analyse_timing.analyse_timing(D, trace_dir)
analyse_trajectory.analyse_trajectory(trace_dir)
analyse_logs.analyse_logs(D, results_dir+'/'+args.trace_name)
analyse_timing.analyse_timing(D, results_dir+'/'+args.trace_name)
if __name__=="__main__":
parser = argparse.ArgumentParser(description='''
Evaluates tracefiles of SVO and generates the plots.
''')
parser.add_argument('experiment_name', help='directory name of the tracefiles')
args = parser.parse_args()
trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'),
'results',
args.experiment_name)
evaluate_dataset(trace_dir)
svo_analysis/src/svo_analysis/analyse_logs.py
+36 -100
View File
@@ -1,16 +1,10 @@
#!/usr/bin/python
import argparse
import csv
import os
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
# tell matplotlib to use latex font
rc('font',**{'family':'serif','serif':['Cardo']})
rc('text', usetex=True)
def analyse_logs(D, trace_name):
def analyse_logs(D, trace_dir):
# identify measurements which result from normal frames and which from keyframes
is_kf = np.argwhere( (D['dropout'] == 1) & (D['repr_n_mps'] >= 0))
@@ -19,114 +13,56 @@ def analyse_logs(D, trace_name):
n_frames = len(is_frame)
n_kfs = len(is_kf)
n_nokf = len(is_nokf)
fps = 25.0
# set initial time to zero
D['timestamp'] = D['timestamp'] - D['timestamp'][0]
# ----------------------------------------------------------------------------
# plot number of reprojected points
mean_n_reproj_points = np.mean(D['repr_n_mps'][is_frame]);
mean_n_reproj_matches = np.mean(D['repr_n_new_references'][is_frame]);
mean_n_edges_final = np.mean(D['sfba_n_edges_final'][is_frame]);
plt.figure()
plt.plot(np.arange(n_frames), D['repr_n_mps'][is_frame], 'r-', label='reprojected points, avg = %(i).2f'%{'i':mean_n_reproj_points})
plt.plot(np.arange(n_frames), D['repr_n_new_references'][is_frame], 'b-', label='reprojected matches, avg = %(i).2f'%{'i':mean_n_reproj_matches})
plt.legend()
plt.tight_layout()
plt.title('Reprojected points asdf')
plt.savefig(trace_name+'_reprojection.pdf', bbox_inches="tight")
# plot number of triangulations
# plt.figure(3)
# plt.plot(np.arange(n_kfs), D['triang_n_points'][is_kf], 'g-', label='triangulated points, avg %0.2f'%np.mean(D['triang_n_points'][is_kf]))
# plt.plot(np.arange(n_kfs), D['triang_n_refs'][is_kf], 'b-', label='new references, avg %0.2f'%np.mean(D['triang_n_refs'][is_kf]))
# plt.plot(np.arange(n_kfs), D['triang_n_fails'][is_kf], 'r-', label='failures, avg %0.2f'%np.mean(D['triang_n_fails'][is_kf]))
# plt.plot(np.arange(n_kfs), D['triang_n_add_refs'][is_kf], 'm-', label='additional refs, avg %0.2f'%np.mean(D['triang_n_add_refs'][is_kf]))
# plt.legend()
# plt.title('Triangulation')
# plt.savefig(trace_name+'_img_triangulation.png', bbox_inches="tight")
# plot number of klt tracked and matched points
fig = plt.figure(figsize=(8,2))
ax = fig.add_subplot(111, xlabel='time [s]', ylabel='no. features')
ax.plot(np.arange(n_frames)/fps, D['repr_n_new_references'][is_frame], 'b-', label='number of successful matches')
fig = plt.figure(figsize=(8,3))
ax = fig.add_subplot(111, xlabel='time [s]')
ax.plot(D['timestamp'][is_frame], D['repr_n_mps'][is_frame], 'r-',
label='Reprojected Points, avg = %.2f'%mean_n_reproj_points)
ax.plot(D['timestamp'][is_frame], D['repr_n_new_references'][is_frame], 'b-',
label='Feature Matches, avg = %.2f'%mean_n_reproj_matches)
ax.plot(D['timestamp'][is_frame], D['sfba_n_edges_final'][is_frame], 'g-',
label='Points after Optimization, avg = %.2f'%mean_n_edges_final)
ax.set_ylim(bottom=0)
ax.legend(loc='lower right')
fig.tight_layout()
fig.savefig(trace_name+'number_of_features.pdf', bbox_inches="tight")
fig.savefig(os.path.join(trace_dir,'num_reprojected.pdf'), bbox_inches="tight")
# ----------------------------------------------------------------------------
# plot median error before and after pose-optimzation and bundle adjustment
init_error_avg = np.mean(D['sfba_error_init'][is_frame])
opt1_avg = np.mean(D['sfba_error_final'][is_frame])
fig = plt.figure(figsize=(8,2))
ax = fig.add_subplot(111, xlabel='time [s]', ylabel='error [px]')
t_max = n_frames
ax.plot(np.arange(n_frames)[0:t_max]/fps, D['sfba_error_init'][is_frame][0:t_max], 'r-', label='Initial error')
ax.plot(np.arange(n_frames)[0:t_max]/fps, D['sfba_error_final'][is_frame][0:t_max], 'b-', label='Final error')
ax.plot(D['timestamp'][is_frame], D['sfba_error_init'][is_frame], 'r-', label='Initial error')
ax.plot(D['timestamp'][is_frame], D['sfba_error_final'][is_frame], 'b-', label='Final error')
ax.legend(ncol=2)
ax.set_xlim([0, t_max/fps+4])
#ax.set_ylim([0, 0.8])
fig.tight_layout()
fig.savefig(trace_name+'reprojection_error.pdf', bbox_inches="tight")
fig.savefig(os.path.join(trace_dir,'reprojection_error.pdf'), bbox_inches="tight")
print 'average reprojection error improvement: ' + str(init_error_avg - opt1_avg)
# plot number of references
plt.figure()
plt.fill(np.arange(n_frames+1), np.append(D['repr_n_new_references'][is_frame]+D['klt_n_matches'][is_frame],0), 'g-', label='Reprojection')
plt.fill(np.arange(n_frames+1), np.append(D['klt_n_tracked'][is_frame],0), 'b-', label='KLT Matches')
plt.plot(np.arange(n_frames), D['sfba_n_edges_final'][is_frame], 'r-', label='References after Optimization', linewidth=2.0)
plt.legend()
plt.tight_layout()
plt.title('Pose Optimization')
plt.savefig(trace_name+'_img_n_references.pdf', bbox_inches="tight")
# ----------------------------------------------------------------------------
# plot number of candidate points
fig = plt.figure(figsize=(8,3))
ax = fig.add_subplot(111, xlabel='time [s]')
ax.plot(D['timestamp'][is_frame], D['n_candidates'][is_frame], 'r-', label='Candidate Points')
fig.tight_layout()
fig.savefig(os.path.join(trace_dir,'candidate_points.pdf'), bbox_inches="tight")
# plot number of seeds
plt.figure()
has_seed = np.argwhere(D['seeds_n_tot'] >= 0)
plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_tot'][has_seed],0), label='nSeeds', edgecolor='none')
plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_updates'][has_seed]+D['seeds_n_failed_matches'][has_seed],0), label='nFailedMatches', edgecolor='none')
plt.fill(np.arange(len(has_seed)+1), np.append(D['seeds_n_updates'][has_seed],0), label='nUpdates', edgecolor='none')
plt.legend()
plt.tight_layout()
plt.title('Seeds')
plt.savefig(trace_name+'_seeds.pdf', bbox_inches="tight")
plt.figure()
plt.plot(D['n_candidates'])
plt.title('Candidates')
plt.show()
# plt.figure()
# plt.plot())
if __name__=="__main__":
dataset='20131003_1937_nslam_i7_asl2_fast'
n_align_frames = 10
save = True
results_dir = '../results/'+dataset
logfile = results_dir+'/'+dataset+'_0.csv'
# parse command line
parser = argparse.ArgumentParser(description='''
This script computes the absolute trajectory error from the ground truth trajectory and the estimated trajectory.
''')
parser.add_argument('--trace_file', help='nanoslam tracefile with timings and logs', default=logfile)
parser.add_argument('--trace_name', help='name of the tracefile, used for saving figures etc.', default=results_dir+'/')
args = parser.parse_args()
# tracefile
data = csv.reader(open(args.trace_file))
fields = data.next()
D = dict()
for field in fields:
D[field] = list()
# fill dictionary with column values
for row in data:
for (field, value) in zip(fields, row):
D[field].append(float(value))
# change dictionary values from list to numpy array for easier manipulation
for field, value in D.items():
D[field] = np.array(D[field])
analyse_logs(D, args.trace_name)
# ----------------------------------------------------------------------------
# plot number of candidate points
fig = plt.figure(figsize=(8,2))
ax = fig.add_subplot(111, xlabel='time [s]', ylabel='px')
ax.plot(D['timestamp'][is_frame], D['sfba_thresh'][is_frame], 'r-', label='Threshold')
fig.tight_layout()
fig.savefig(os.path.join(trace_dir,'optimization_thresh.pdf'), bbox_inches="tight")
svo_analysis/src/svo_analysis/analyse_timing.py
+40 -69
View File
@@ -1,97 +1,68 @@
#!/usr/bin/python
import argparse
import csv
import os
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
def analyse_timing(D, trace_name):
def analyse_timing(D, trace_dir):
# identify measurements which result from normal frames and which from keyframes
is_frame = np.argwhere(D['repr_n_mps'] >= 0)
n_frames = len(is_frame)
# plot total time for frame processing
avg_time = np.mean(D['t_tot_time'][is_frame])*1000;
# set initial time to zero
D['timestamp'] = D['timestamp'] - D['timestamp'][0]
# ----------------------------------------------------------------------------
# plot total time for frame processing
avg_time = np.mean(D['tot_time'][is_frame])*1000;
fig = plt.figure(figsize=(8, 3))
ax = fig.add_subplot(111, ylabel='processing time [ms]', xlabel='time [s]')
ax.plot(np.arange(n_frames), D['t_tot_time'][is_frame]*1000, 'g-', label='total time [ms]')
ax.plot(np.arange(n_frames), np.ones(n_frames)*avg_time, 'b--', label=str('%(time).1fms mean time' % {'time': avg_time}))
ax.plot(D['timestamp'][is_frame], D['tot_time'][is_frame]*1000, 'g-', label='total time [ms]')
ax.plot(D['timestamp'][is_frame], np.ones(n_frames)*avg_time, 'b--', label=str('%(time).1fms mean time' % {'time': avg_time}))
ax.legend()
fig.tight_layout()
fig.savefig(trace_name+'_img_tot_processing_time.pdf', bbox_inches="tight")
fig.savefig(os.path.join(trace_dir,'timing.pdf'), bbox_inches="tight")
# ----------------------------------------------------------------------------
# plot boxplot
fig = plt.figure(figsize=(6,2))
ax = fig.add_subplot(111, xlabel='Processing time [ms]')
ax.boxplot([
D['t_tot_time'][is_frame]*1000,
D['tot_time'][is_frame]*1000,
# D['t_local_ba'][is_kf]*1000,
D['t_pose_optimizer'][is_frame]*1000 + D['t_point_optimizer'][is_frame]*1000,
D['t_reproject'][is_frame]*1000,
D['t_sparse_img_align'][is_frame]*1000,
D['t_pyramid_creation'][is_frame]*1000
], 0,'', vert=0) #
D['pose_optimizer'][is_frame]*1000 + D['point_optimizer'][is_frame]*1000,
D['reproject'][is_frame]*1000,
D['sparse_img_align'][is_frame]*1000,
D['pyramid_creation'][is_frame]*1000
], 0,'', vert=0)
boxplot_labels = [
r'\textbf{Total Motion Estimation: %2.2fms}' % np.median(D['t_tot_time'][is_frame]*1000),
# 'Local BA (KF only): %.2fms ' % np.median(D['t_local_ba'][is_kf]*1000),
'Refinement: %2.2fms' % np.median(D['t_pose_optimizer'][is_frame]*1000 + D['t_point_optimizer'][is_frame]*1000),
'Feature Alignment: %2.2fms' % np.median(D['t_reproject'][is_frame]*1000),
'Sparse Image Alignment: %2.2fms' % np.median(D['t_sparse_img_align'][is_frame]*1000),
'Pyramid Creation: %2.2fms' % np.median(D['t_pyramid_creation'][is_frame]*1000) ]
r'\textbf{Total Motion Estimation: %2.2fms}' % np.median(D['tot_time'][is_frame]*1000),
# 'Local BA (KF only): %.2fms ' % np.median(D['local_ba'][is_kf]*1000),
'Refinement: %2.2fms' % np.median(D['pose_optimizer'][is_frame]*1000 + D['point_optimizer'][is_frame]*1000),
'Feature Alignment: %2.2fms' % np.median(D['reproject'][is_frame]*1000),
'Sparse Image Alignment: %2.2fms' % np.median(D['sparse_img_align'][is_frame]*1000),
'Pyramid Creation: %2.2fms' % np.median(D['pyramid_creation'][is_frame]*1000) ]
ax.set_yticks(np.arange(len(boxplot_labels))+1)
ax.set_yticklabels(boxplot_labels)
fig.tight_layout()
fig.savefig(trace_name+'_timing_boxplot.pdf', bbox_inches="tight")
fig.savefig(os.path.join(trace_dir,'timing_boxplot.pdf'), bbox_inches="tight")
fig = plt.figure(figsize=(8, 5))
ax = fig.add_subplot(111, ylabel='time [ms]')
ax.plot(D['t_reproject']*1000, 'r', label='reproject')
ax.plot(D['t_reproject_find_pts']*1000, 'g', label='reproject find pts') # ok!
ax.plot(D['t_reproject_best']*1000, 'm', label='reproject best')
ax.plot(D['t_reproject_candidates']*1000, 'b', label='reproject candidates')
ax.legend()
# ----------------------------------------------------------------------------
# plot boxplot reprojection
fig = plt.figure(figsize=(6,2))
ax = fig.add_subplot(111, xlabel='Processing time [ms]')
ax.boxplot([ D['reproject'][is_frame]*1000,
D['feature_align'][is_frame]*1000,
D['reproject_candidates'][is_frame]*1000,
D['reproject_kfs'][is_frame]*1000 ], 0, '', vert=0)
boxplot_labels = [r'\textbf{Total Reprojection: %2.2fms}' % np.median(D['reproject'][is_frame]*1000),
'Feature Alignment: %2.2fms' % np.median(D['feature_align'][is_frame]*1000),
'Reproject Candidates: %2.2fms' % np.median(D['reproject_candidates'][is_frame]*1000),
'Reproject Keyframes: %2.2fms' % np.median(D['reproject_kfs'][is_frame]*1000) ]
ax.set_yticks(np.arange(len(boxplot_labels))+1)
ax.set_yticklabels(boxplot_labels)
fig.tight_layout()
fig.savefig(trace_name+'_reprojection.pdf', bbox_inches="tight")
if __name__=="__main__":
# config
dataset='20131003_1937_nslam_i7_asl2_fast'
results_dir = '../results/'+dataset
tracefile = results_dir + '/'+dataset+'_0.csv'
# parse command line
parser = argparse.ArgumentParser(description='''
This script computes the absolute trajectory error from the ground truth trajectory and the estimated trajectory.
''')
parser.add_argument('--trace_file', help='nanoslam tracefile with timings and logs', default=tracefile)
parser.add_argument('--trace_name', help='name of the tracefile, used for saving figures etc.', default=results_dir)
args = parser.parse_args()
results_dir = args.trace_name
# tracefile
data = csv.reader(open(args.trace_file))
fields = data.next()
D = dict()
for field in fields:
D[field] = list()
# fill dictionary with column values
for row in data:
for (field, value) in zip(fields, row):
D[field].append(float(value))
# change dictionary values from list to numpy array for easier manipulation
for field, value in D.items():
D[field] = np.array(D[field])
# tell matplotlib to use latex font
rc('font',**{'family':'serif','serif':['Cardo']})
rc('text', usetex=True)
fig.savefig(os.path.join(trace_dir,'timing_reprojection.pdf'), bbox_inches="tight")