From 8be2d1cb353ade9f7b2e473153fb1a7a597924ea Mon Sep 17 00:00:00 2001 From: Christian Forster Date: Wed, 9 Apr 2014 20:09:01 +0200 Subject: [PATCH] benchmark multiple scenes in one go --- svo_analysis/experiments/synthetic.yaml | 7 ++ svo_analysis/scripts/benchmark.py | 115 ++++++++++++------------ 2 files changed, 66 insertions(+), 56 deletions(-) create mode 100644 svo_analysis/experiments/synthetic.yaml diff --git a/svo_analysis/experiments/synthetic.yaml b/svo_analysis/experiments/synthetic.yaml new file mode 100644 index 0000000..5eb0402 --- /dev/null +++ b/svo_analysis/experiments/synthetic.yaml @@ -0,0 +1,7 @@ +experiment_label: 'svo fast' +param_settings: vo_fast +datasets: + - 1404091756_fr1_straight + - 1404091807_fr1_straight_rot + - 1404091819_fr1_sine_2d + - 1404091831_fr1_sine_3d \ No newline at end of file diff --git a/svo_analysis/scripts/benchmark.py b/svo_analysis/scripts/benchmark.py index 3be6916..9611350 100755 --- a/svo_analysis/scripts/benchmark.py +++ b/svo_analysis/scripts/benchmark.py @@ -14,67 +14,70 @@ import vikit_py.cpu_info as cpu_info import vikit_py.ros_node as ros_node import evaluate +def run_experiment(dataset, params): + # load dataset parameters + params['dataset_directory'] = os.path.join(os.environ['SVO_DATASET_DIR'], dataset) + if not os.path.exists(params['dataset_directory']): + raise Exception("Provided dataset folder does not exist.") + dataset_params_file = os.path.join(params['dataset_directory'], 'dataset_params.yaml') + dataset_params = yaml.load(open(dataset_params_file,'r')) + + # load algorithm parameters + algo_params_file = os.path.join(rospkg.RosPack().get_path('svo_ros'), + 'param', params['param_settings']+'.yaml') + algo_params = yaml.load(open(algo_params_file,'r')) + + # combine all parameters + params = dict(params.items() + algo_params.items() + dataset_params['rig']['cam0'].items()) + + # dump experiment params to file: + params_dump_file = params['trace_dir']+'/params.yaml' + with open(params_dump_file,'w') as outfile: + outfile.write(yaml.dump(params, default_flow_style=False)) + + # execute ros node + node = ros_node.RosNode(args.version, args.executable) + node.run(params) + if __name__=="__main__": - # parse command line - parser = argparse.ArgumentParser(description=''' - Runs SVO with the dataset and parameters specified in the provided experiment file. - ''') - parser.add_argument('experiment_file', help='experiment file in svo_analysis/experiments folder') - parser.add_argument('--name', help='experiment name') - parser.add_argument('--evaluate', help='evaluate tracefile after running SVO', action='store_true') - parser.add_argument('--version', help='version of svo to evaluate', default='svo_ros') - parser.add_argument('--executable', help='the executable to be called', default='benchmark') - args = parser.parse_args() - - # load experiment parameters - expParamsFile = os.path.join(rospkg.RosPack().get_path('svo_analysis'), + # parse command line + parser = argparse.ArgumentParser(description=''' + Runs SVO with the dataset and parameters specified in the provided experiment file. + ''') + parser.add_argument('experiment_file', help='experiment file in svo_analysis/experiments folder') + parser.add_argument('--evaluate', help='evaluate tracefile after running SVO', action='store_true') + parser.add_argument('--version', help='version of svo to evaluate', default='svo_ros') + parser.add_argument('--executable', help='the executable to be called', default='benchmark') + args = parser.parse_args() + + # load experiment parameters + experiment_params_file = os.path.join(rospkg.RosPack().get_path('svo_analysis'), 'experiments', args.experiment_file+'.yaml') - expParams = yaml.load(open(expParamsFile, 'r')) - expParams['time'] = time.strftime("%Y%m%d_%H%M", time.localtime()) - expParams['platform'] = cpu_info.get_cpu_info() - expParams['experiment_name'] = args.name - if(expParams['experiment_name'] == None): - expParams['experiment_name'] = expParams['time']+'_'+args.version+'_' + args.experiment_file - - # load dataset parameters - expParams['dataset_directory'] = os.path.join(os.environ['SVO_DATASET_DIR'], expParams['dataset']) - if not os.path.exists(expParams['dataset_directory']): - raise Exception("Provided dataset folder does not exist.") - datasetParamsFile = os.path.join(expParams['dataset_directory'], 'dataset_params.yaml') - datasetParams = yaml.load(open(datasetParamsFile,'r')) - - # load algorithm parameters - algoParamsFile = os.path.join(rospkg.RosPack().get_path('svo_ros'), - 'param', expParams['param_settings']+'.yaml') - algoParams = yaml.load(open(algoParamsFile,'r')) - - # combine all parameters - params = dict(expParams.items() + datasetParams.items() + datasetParams['rig']['cam0'].items() + algoParams.items()) - params['trace_name'] = 'trace' - params['trace_dir'] = os.path.join(rospkg.RosPack().get_path('svo_analysis'), - 'results', params['experiment_name']) - + experiment_params = yaml.load(open(experiment_params_file, 'r')) + params = dict() + params['experiment_label'] = experiment_params['experiment_label'] + params['param_settings'] = experiment_params['param_settings'] + params['time'] = time.strftime("%Y%m%d_%H%M", time.localtime()) + params['platform'] = cpu_info.get_cpu_info() + params['trace_name'] = 'trace' + + trace_dirs = list() + for dataset in experiment_params['datasets']: + params['experiment_name'] = params['time']+'_'+args.version+'_'+args.experiment_file+'_'+dataset + params['trace_dir'] = os.path.join(rospkg.RosPack().get_path('svo_analysis'), + 'results', params['experiment_name']) + if not os.path.exists(params['trace_dir']): + os.makedirs(params['trace_dir']) + trace_dirs.append(params['trace_dir']) + run_experiment(dataset, params) + + # TODO: check if it is a synthetic dataset + for trace in trace_dirs: + evaluate.evaluate_dataset(trace) + - # create folder for tracing - if not os.path.exists(params['trace_dir']): - os.makedirs(params['trace_dir']) - - # dump experiment params to file: - params_dump_file = params['trace_dir']+'/params.yaml' - with open(params_dump_file,'w') as outfile: - outfile.write(yaml.dump(params, default_flow_style=False)) - - # execute ros node - node = ros_node.RosNode(args.version, args.executable) - node.run(params) - - # TODO: check if it is a synthetic dataset - evaluate.evaluate_dataset(params['trace_dir']) - - -