evaluation scripts

svo_analysis/scripts/comparison.py

@@ -8,6 +8,7 @@ import numpy as np
 import matplotlib.pyplot as plt
 import yaml
 import argparse
+import svo_analysis.analyse_depth as analyse_depth
 from matplotlib import rc
 
 # tell matplotlib to use latex font
@@ -43,6 +44,7 @@ def compare_results(comp_params, results_dir, comparison_dir):
     print('run comparison: '+comp_params['comparison_name'])
            
     line_styles = ['-','--',':']
+    line_colors = ['b','g','r','m','c']
     
     # -------------------------------------------------------------------------
     # plot trajectory:
@@ -57,10 +59,9 @@ def compare_results(comp_params, results_dir, comparison_dir):
                 base_plot, = ax.plot(data[:,1], data[:,2], label=exp_set['label'], linestyle=line_styles[np.mod(i, len(line_styles))])
             else:
                 ax.plot(data[:,1], data[:,2], color=base_plot.get_color(), linestyle= line_styles[np.mod(i, len(line_styles))])
-    ax.legend(loc='upper left', ncol=2)
+    ax.legend(loc='upper left', ncol=3)
     save_figure(fig, 'trajectory', comparison_dir)
     
-    
     # -------------------------------------------------------------------------
     # plot translation error:
     fig = plt.figure(figsize=(6,2))
@@ -93,9 +94,28 @@ def compare_results(comp_params, results_dir, comparison_dir):
             else:
                 ax.plot(distances, e*1000, color=base_plot.get_color(), linestyle= line_styles[np.mod(i, len(line_styles))])
                 
-    ax.legend(loc='upper left', ncol=2)
+    ax.legend(loc='upper left', ncol=3)
     save_figure(fig, 'translation_error', comparison_dir)
     
+    # -------------------------------------------------------------------------
+    # plot depth estimation error:
+    fig = plt.figure(figsize=(6,5))
+    ax = fig.add_subplot(111, xlabel='Error [m]', ylabel='Precision [\%]')
+    for k, exp_set in enumerate(comp_params['experiment_sets']):
+        print('plot depth error for experiment: ' + exp_set['label'])
+        
+        for i, exp in enumerate(exp_set['experiments']):
+            
+            D = np.loadtxt(os.path.join(results_dir, exp, 'depth_error.txt'), delimiter=' ')
+            D = D[np.logical_and(D[:,0] > 1, D[:,0] < D[-1,0]/4),:]
+            errors = np.abs(D[:,1])
+            if i == 0:
+                base_plot = analyse_depth.precision_plot(ax, errors, 0.05, line_colors[k], line_styles[np.mod(i, len(line_styles))])
+            else: 
+                base_plot = analyse_depth.precision_plot(ax, errors, 0.05, base_plot.get_color(), line_styles[np.mod(i, len(line_styles))])
+                
+    save_figure(fig, 'depth_error', comparison_dir)
+  
 if __name__ == '__main__':
   
     parser = argparse.ArgumentParser(description='Compare results of a VO pipeline.')

svo_analysis/scripts/evaluate.py

@@ -10,41 +10,49 @@ import csv
 import numpy as np
 import rospkg
 import argparse
+import yaml
 import svo_analysis.analyse_logs as analyse_logs
 import svo_analysis.analyse_timing as analyse_timing
 import svo_analysis.analyse_trajectory as analyse_trajectory
+import svo_analysis.analyse_depth as analyse_depth
 
 def evaluate_dataset(trace_dir):
   
-  # read tracefile
+    # read tracefile
-  data = csv.reader(open(os.path.join(trace_dir, 'trace.csv')))
+    data = csv.reader(open(os.path.join(trace_dir, 'trace.csv')))
-  fields = data.next()
+    fields = data.next()
-  D = dict()
+    D = dict()
-  for field in fields:
+    for field in fields:
-    D[field] = list()
+        D[field] = list()
 
-  # fill dictionary with column values
+    # fill dictionary with column values
-  for row in data:
+    for row in data:
-    for (field, value) in zip(fields, row):
+        for (field, value) in zip(fields, row):
-      D[field].append(float(value))
+          D[field].append(float(value))
 
-  # change dictionary values from list to numpy array for easier manipulation
+    # change dictionary values from list to numpy array for easier manipulation
-  for field, value in D.items():
+    for field, value in D.items():
-    D[field] = np.array(D[field])
+        D[field] = np.array(D[field])
+
+    # generate plots
+    analyse_logs.analyse_logs(D, trace_dir)
+    analyse_timing.analyse_timing(D, trace_dir)
+    analyse_trajectory.analyse_trajectory(trace_dir)
+
+    param = yaml.load(open(os.path.join(trace_dir, 'dataset_params.yaml'), 'r'))
+    if param['dataset_is_blender']:
+        analyse_depth.analyse_depth(trace_dir)
+        analyse_depth.analyse_depth_over_time(trace_dir)
 
-  # generate plots
-  analyse_logs.analyse_logs(D, trace_dir)
-  analyse_timing.analyse_timing(D, trace_dir)
-  analyse_trajectory.analyse_trajectory(trace_dir)
 
 if __name__=="__main__":
-  parser = argparse.ArgumentParser(description='''
+    parser = argparse.ArgumentParser(description='''
-  Evaluates tracefiles of SVO and generates the plots.
+    Evaluates tracefiles of SVO and generates the plots.
-  ''')
+    ''')
-  parser.add_argument('experiment_name', help='directory name of the tracefiles')
+    parser.add_argument('experiment_name', help='directory name of the tracefiles')
-  args = parser.parse_args()
+    args = parser.parse_args()
 
-  trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'), 
+    trace_dir = os.path.join(rospkg.RosPack().get_path('svo_analysis'), 
-                           'results',
+                             'results',
-                           args.experiment_name)
+                             args.experiment_name)
-  evaluate_dataset(trace_dir)
+    evaluate_dataset(trace_dir)

svo_analysis/src/svo_analysis/analyse_logs.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import os
+import yaml
 import numpy as np
 import matplotlib.pyplot as plt
   
@@ -10,9 +11,6 @@ def analyse_logs(D, trace_dir):
   is_kf = np.argwhere( (D['dropout'] == 1) & (D['repr_n_mps'] >= 0))
   is_frame = np.argwhere(D['repr_n_mps'] >= 0)
   is_nokf = np.argwhere( (D['dropout'] == 0) & (D['repr_n_mps'] >= 0))
-  n_frames = len(is_frame)
-  n_kfs = len(is_kf)
-  n_nokf = len(is_nokf)
    
   # set initial time to zero
   D['timestamp'] = D['timestamp'] - D['timestamp'][0]
@@ -66,3 +64,11 @@ def analyse_logs(D, trace_dir):
   fig.tight_layout()
   fig.savefig(os.path.join(trace_dir,'optimization_thresh.pdf'), bbox_inches="tight")
   
+  # ----------------------------------------------------------------------------
+  # write other statistics to file
+  stat = {'num_frames': len(is_frame), 
+          'num_kfs': len(is_kf),
+          'reproj_error_avg_improvement': float(init_error_avg - opt1_avg)}
+  with open(os.path.join(trace_dir,'dataset_stats.yaml'),'w') as outfile:          
+    outfile.write(yaml.dump(stat, default_flow_style=False))
+

svo_analysis/src/svo_analysis/analyse_trajectory.py

@@ -100,7 +100,7 @@ def analyse_trajectory(results_dir, n_align_frames = 200, use_hand_eye_calib = T
         ax = fig.add_subplot(111, title='trajectory', aspect='equal', xlabel='x [m]', ylabel='y [m]')
         ax.plot(p_es_aligned[:,0], p_es_aligned[:,1], 'b-', label='estimate')
         ax.plot(p_gt[:,0], p_gt[:,1], 'r-', label='groundtruth')
-        #ax.plot(p_es_aligned[0:n_align_frames,0], p_es_aligned[0:n_align_frames,1], 'g-', linewidth=2, label='aligned')
+        ax.plot(p_es_aligned[0:n_align_frames,0], p_es_aligned[0:n_align_frames,1], 'g-', linewidth=2, label='aligned')
         ax.legend()
         fig.tight_layout()
         fig.savefig(results_dir+'/trajectory.pdf')