Mercurial > repos > yating-l > data_manager_fetch_reference_data

comparison data_manager/fetch_reference_data.py @ 5:2f926e7d623d draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload
author yating-l
date Wed, 03 May 2017 17:43:57 -0400
parents 464d75111b16
children 2eb398f3649c
comparison
equal deleted inserted replaced
4:464d75111b16 5:2f926e7d623d
24 def get_reference_id_name(params): 24 def get_reference_id_name(params):
25 genome_id = params['param_dict']['genome_id'] 25 genome_id = params['param_dict']['genome_id']
26 genome_name = params['param_dict']['genome_name'] 26 genome_name = params['param_dict']['genome_name']
27 return genome_id, genome_name 27 return genome_id, genome_name
28 28
29 def download_from_GlimmerHMM(data_manager_dict, params, target_directory, sequence_id, sequence_name ): 29 def get_url(params):
30 GlimmerHMM_DOWNLOAD_URL = 'ftp://ccb.jhu.edu/pub/software/glimmerhmm/GlimmerHMM-3.0.4.tar.gz' 30 trained_url = params['param_dict']['trained_url']
31 GlimmerHMM_TRAINED_DIR = os.path.join('GlimmerHMM', 'trained_dir', sequence_id) 31 return trained_url
32 with tarfile.open('GlimmerHMM-3.0.4.tar', mode='r:*') as tar: 32
33 def download_from_GlimmerHMM(data_manager_dict, target_directory, sequence_id, sequence_name, trained_dir):
34 if not trained_dir:
35 trained_dir = 'ftp://ccb.jhu.edu/pub/software/glimmerhmm/GlimmerHMM-3.0.4.tar.gz'
36 #Download trained data, ref: https://dzone.com/articles/how-download-file-python
37 f = urllib2.urlopen(trained_dir)
38 data = f.read()
39 downloadpath = 'tmp'
40 os.mkdir(downloadpath)
41 filepath = os.path.join(downloadpath, 'GlimmerHMM-3.0.4.tar')
42 with open(filepath, 'wb') as code:
43 code.write(data)
44 with tarfile.open(filepath, mode='r:*') as tar:
33 subdir = [ 45 subdir = [
34 tarinfo for tarinfo in tar.getmembers() 46 tarinfo for tarinfo in tar.getmembers()
35 if sequence_id in tarinfo.name 47 if sequence_id in tarinfo.name
36 ] 48 ]
37 tar.extractall(members=subdir) 49 tar.extractall(members=subdir)
50 GlimmerHMM_TRAINED_DIR = os.path.join(downloadpath, 'GlimmerHMM', 'trained_dir', sequence_id)
38 glimmerhmm_trained_target_dir = os.path.join(target_directory, sequence_id) 51 glimmerhmm_trained_target_dir = os.path.join(target_directory, sequence_id)
39 shutil.copytree(GlimmerHMM_TRAINED_DIR, glimmerhmm_trained_target_dir) 52 shutil.copytree(GlimmerHMM_TRAINED_DIR, glimmerhmm_trained_target_dir)
40 data_table_entry = dict(value=sequence_id, name=sequence_name, path=glimmerhmm_trained_target_dir) 53 data_table_entry = dict(value=sequence_id, name=sequence_name, path=glimmerhmm_trained_target_dir)
41 _add_data_table_entry(data_manager_dict, data_table_entry) 54 _add_data_table_entry(data_manager_dict, data_table_entry)
55
56 cleanup_before_exit('tmp')
42 57
43 cleanup_before_exit(GlimmerHMM_TRAINED_DIR) 58 def _add_data_table_entry(data_manager_dict, data_table_entry):
44
45 def _add_data_table_entry( data_manager_dict, data_table_entry ):
46 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} ) 59 data_manager_dict['data_tables'] = data_manager_dict.get( 'data_tables', {} )
47 data_manager_dict['data_tables']['reference_data'] = data_manager_dict['data_tables'].get('reference_data', []) 60 data_manager_dict['data_tables']['reference_data'] = data_manager_dict['data_tables'].get('reference_data', [])
48 data_manager_dict['data_tables']['reference_data'].append( data_table_entry ) 61 data_manager_dict['data_tables']['reference_data'].append( data_table_entry )
49 return data_manager_dict 62 return data_manager_dict
50
51 REFERENCE_SOURCE_TO_DOWNLOAD = dict(glimmerhmm=download_from_GlimmerHMM)
52 63
53 def main(): 64 def main():
54 #Parse Command Line 65 #Parse Command Line
55 parser = argparse.ArgumentParser() 66 parser = argparse.ArgumentParser()
56 parser.add_argument('-o', '--out', help='Output file') 67 parser.add_argument('-o', '--out', help='Output file')
62 target_directory = params['output_data'][0]['extra_files_path'] 73 target_directory = params['output_data'][0]['extra_files_path']
63 os.mkdir(target_directory) 74 os.mkdir(target_directory)
64 data_manager_dict = {} 75 data_manager_dict = {}
65 76
66 sequence_id, sequence_name = get_reference_id_name(params) 77 sequence_id, sequence_name = get_reference_id_name(params)
67 78 trained_dir = get_url(params)
68 #Fetch the FASTA 79 #Fetch the FASTA
69 REFERENCE_SOURCE_TO_DOWNLOAD[params['param_dict']['trained_dir']](data_manager_dict, params, target_directory, sequence_id, sequence_name) 80 download_from_GlimmerHMM(data_manager_dict, target_directory, sequence_id, sequence_name, trained_dir)
70
71 #save info to json file 81 #save info to json file
72 open(filename, 'wb').write(to_json_string(data_manager_dict)) 82 open(filename, 'wb').write(to_json_string(data_manager_dict))
73 83
74 if __name__ == "__main__": 84 if __name__ == "__main__":
75 main() 85 main()