Mercurial > repos > greg > data_manager_vsnp_excel

comparison data_manager/vsnp_excel_fetcher.py @ 1:54389d791983 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author greg
date Mon, 10 Feb 2020 13:06:19 +0000
parents f7183dcf0430
children
comparison
equal deleted inserted replaced
0:f7183dcf0430 1:54389d791983
1 import argparse 1 import argparse
2 import json 2 import json
3 import os 3 import os
4 import sys
4 try: 5 try:
5 # For Python 3.0 and later 6 # For Python 3.0 and later
6 from urllib.request import Request, urlopen 7 from urllib.request import Request, urlopen
7 except ImportError: 8 except ImportError:
8 # Fall back to Python 2 imports 9 # Fall back to Python 2 imports
9 from urllib2 import Request, urlopen 10 from urllib2 import Request, urlopen
10 11
11 12
12 def url_download(url, target_directory): 13 def url_download(url, workdir):
13 file_path = os.path.abspath(os.path.join(target_directory, os.path.basename(url))) 14 file_path = os.path.abspath(os.path.join(workdir, os.path.basename(url)))
14 src = None 15 src = None
15 dst = None 16 dst = None
16 try: 17 try:
17 req = Request(url) 18 req = Request(url)
18 src = urlopen(req) 19 src = urlopen(req)
21 chunk = src.read(2**10) 22 chunk = src.read(2**10)
22 if chunk: 23 if chunk:
23 dst.write(chunk) 24 dst.write(chunk)
24 else: 25 else:
25 break 26 break
27 except Exception as e:
28 sys.exit(str(e))
26 finally: 29 finally:
27 if src: 30 if src:
28 src.close() 31 src.close()
29 return file_path 32 return file_path
30 33
31 34
32 def download(dbkey, display_name, url, out_file): 35 def download(dbkey, name, url, out_file):
33 36
34 with open(out_file) as fh: 37 with open(out_file) as fh:
35 params = json.loads(fh.read()) 38 params = json.loads(fh.read())
36 39
37 work_directory = params['output_data'][0]['extra_files_path'] 40 workdir = params['output_data'][0]['extra_files_path']
38 target_directory = os.path.abspath(os.path.join(work_directory, "vsnp", dbkey)) 41 os.makedirs(workdir)
39 os.mkdir(target_directory) 42 file_path = url_download(url, workdir)
40 file_path = url_download(url, target_directory) 43 entry_name = os.path.basename(file_path)
41 entry_name = "%s" % os.path.basename(file_path)
42 44
43 data_manager_json = {"data_tables": {}} 45 data_manager_json = {"data_tables": {}}
44 data_manager_entry = {} 46 data_manager_entry = {}
45 data_manager_entry['value'] = dbkey 47 data_manager_entry['value'] = dbkey
46 data_manager_entry['name'] = entry_name 48 data_manager_entry['name'] = entry_name
47 data_manager_entry['path'] = file_path 49 data_manager_entry['path'] = file_path
48 data_manager_entry['description'] = "Excel file for %s" % display_name 50 data_manager_entry['description'] = "Excel file for %s" % name
49 data_manager_json["data_tables"]["vsnp_excel"] = data_manager_entry 51 data_manager_json["data_tables"]["vsnp_excel"] = data_manager_entry
50 52
51 with open(out_file, 'w') as fh: 53 with open(out_file, 'w') as fh:
52 fh.write(json.dumps(data_manager_json, sort_keys=True)) 54 fh.write(json.dumps(data_manager_json, sort_keys=True))
53 55
54 56
55 parser = argparse.ArgumentParser() 57 parser = argparse.ArgumentParser()
56 58
57 parser.add_argument('--dbkey', dest='dbkey', help='Genome reference dbkey') 59 parser.add_argument('--dbkey', dest='dbkey', help='Genome reference dbkey')
58 parser.add_argument('--display_name', dest='display_name', help='Reference display name') 60 parser.add_argument('--name', dest='name', help='Reference display name')
59 parser.add_argument('--url', dest='url', help='URL to download Excel file') 61 parser.add_argument('--url', dest='url', help='URL to download Excel file')
60 parser.add_argument('--out_file', dest='out_file', help='JSON output file') 62 parser.add_argument('--out_file', dest='out_file', help='JSON output file')
61 63
62 args = parser.parse_args() 64 args = parser.parse_args()
63 65
64 download(args.dbkey, args.display_name, args.url, args.out_file) 66 download(args.dbkey, args.name, args.url, args.out_file)