Mercurial > repos > rnateam > dorina
annotate data_source.py @ 1:32c27a1d81a4 draft default tip
planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/rna_tools/dorina/ commit d720bfabe099baad94dfe54442cb6a4dd023524a
| author | rnateam |
|---|---|
| date | Fri, 07 Oct 2016 09:43:28 -0400 |
| parents | f388dcb96df8 |
| children |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 # Retrieves data from external data source applications and stores in a dataset file. | |
| 3 # Data source application parameters are temporarily stored in the dataset file. | |
| 4 import socket, urllib, sys, os | |
| 5 from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg... | |
| 6 from galaxy.util import get_charset_from_http_headers | |
| 7 import galaxy.model # need to import model before sniff to resolve a circular import dependency | |
| 8 from galaxy.datatypes import sniff | |
| 9 from galaxy.datatypes.registry import Registry | |
| 10 from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE | |
|
1
32c27a1d81a4
planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/rna_tools/dorina/ commit d720bfabe099baad94dfe54442cb6a4dd023524a
rnateam
parents:
0
diff
changeset
|
11 from json import loads, dumps |
| 0 | 12 |
| 13 assert sys.version_info[:2] >= ( 2, 4 ) | |
| 14 | |
| 15 def stop_err( msg ): | |
| 16 sys.stderr.write( msg ) | |
| 17 sys.exit() | |
| 18 | |
| 19 GALAXY_PARAM_PREFIX = 'GALAXY' | |
| 20 GALAXY_ROOT_DIR = os.path.realpath( os.path.join( os.path.split( os.path.realpath( __file__ ) )[0], '..', '..' ) ) | |
| 21 GALAXY_DATATYPES_CONF_FILE = os.path.join( GALAXY_ROOT_DIR, 'datatypes_conf.xml' ) | |
| 22 | |
| 23 def load_input_parameters( filename, erase_file = True ): | |
| 24 datasource_params = {} | |
| 25 try: | |
| 26 json_params = loads( open( filename, 'r' ).read() ) | |
| 27 datasource_params = json_params.get( 'param_dict' ) | |
| 28 except: | |
| 29 json_params = None | |
| 30 for line in open( filename, 'r' ): | |
| 31 try: | |
| 32 line = line.strip() | |
| 33 fields = line.split( '\t' ) | |
| 34 datasource_params[ fields[0] ] = fields[1] | |
| 35 except: | |
| 36 continue | |
| 37 if erase_file: | |
| 38 open( filename, 'w' ).close() #open file for writing, then close, removes params from file | |
| 39 return json_params, datasource_params | |
| 40 | |
| 41 def __main__(): | |
| 42 filename = sys.argv[1] | |
| 43 try: | |
| 44 max_file_size = int( sys.argv[2] ) | |
| 45 except: | |
| 46 max_file_size = 0 | |
| 47 | |
| 48 job_params, params = load_input_parameters( filename ) | |
| 49 | |
| 50 if job_params is None: #using an older tabular file | |
| 51 enhanced_handling = False | |
| 52 job_params = dict( param_dict = params ) | |
| 53 job_params[ 'output_data' ] = [ dict( out_data_name = 'output', | |
| 54 ext = 'data', | |
| 55 file_name = filename, | |
| 56 extra_files_path = None ) ] | |
| 57 job_params[ 'job_config' ] = dict( GALAXY_ROOT_DIR=GALAXY_ROOT_DIR, GALAXY_DATATYPES_CONF_FILE=GALAXY_DATATYPES_CONF_FILE, TOOL_PROVIDED_JOB_METADATA_FILE = TOOL_PROVIDED_JOB_METADATA_FILE ) | |
| 58 else: | |
| 59 enhanced_handling = True | |
| 60 json_file = open( job_params[ 'job_config' ][ 'TOOL_PROVIDED_JOB_METADATA_FILE' ], 'w' ) #specially named file for output junk to pass onto set metadata | |
| 61 | |
| 62 datatypes_registry = Registry() | |
| 63 datatypes_registry.load_datatypes( root_dir = job_params[ 'job_config' ][ 'GALAXY_ROOT_DIR' ], config = job_params[ 'job_config' ][ 'GALAXY_DATATYPES_CONF_FILE' ] ) | |
| 64 | |
| 65 URL = params.get( 'URL', None ) #using exactly URL indicates that only one dataset is being downloaded | |
| 66 URL_method = params.get( 'URL_method', None ) | |
| 67 | |
| 68 # The Python support for fetching resources from the web is layered. urllib uses the httplib | |
| 69 # library, which in turn uses the socket library. As of Python 2.3 you can specify how long | |
| 70 # a socket should wait for a response before timing out. By default the socket module has no | |
| 71 # timeout and can hang. Currently, the socket timeout is not exposed at the httplib or urllib2 | |
| 72 # levels. However, you can set the default timeout ( in seconds ) globally for all sockets by | |
| 73 # doing the following. | |
| 74 socket.setdefaulttimeout( 600 ) | |
| 75 | |
| 76 for data_dict in job_params[ 'output_data' ]: | |
| 77 cur_filename = data_dict.get( 'file_name', filename ) | |
| 78 cur_URL = params.get( '%s|%s|URL' % ( GALAXY_PARAM_PREFIX, data_dict[ 'out_data_name' ] ), URL ) | |
| 79 if not cur_URL: | |
| 80 open( cur_filename, 'w' ).write( "" ) | |
| 81 stop_err( 'The remote data source application has not sent back a URL parameter in the request.' ) | |
| 82 | |
| 83 # The following calls to urllib.urlopen() will use the above default timeout | |
| 84 try: | |
| 85 if not URL_method or URL_method == 'get': | |
| 86 page = urllib.urlopen( cur_URL ) | |
| 87 elif URL_method == 'post': | |
| 88 page = urllib.urlopen( cur_URL, urllib.urlencode( params ) ) | |
| 89 except Exception, e: | |
| 90 stop_err( 'The remote data source application may be off line, please try again later. Error: %s' % str( e ) ) | |
| 91 if max_file_size: | |
| 92 file_size = int( page.info().get( 'Content-Length', 0 ) ) | |
| 93 if file_size > max_file_size: | |
| 94 stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) ) | |
| 95 #do sniff stream for multi_byte | |
| 96 try: | |
| 97 cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) ) | |
| 98 except Exception, e: | |
| 99 stop_err( 'Unable to fetch %s:\n%s' % ( cur_URL, e ) ) | |
| 100 | |
| 101 #here import checks that upload tool performs | |
| 102 if enhanced_handling: | |
| 103 try: | |
| 104 ext = sniff.handle_uploaded_dataset_file( filename, datatypes_registry, ext = data_dict[ 'ext' ], is_multi_byte = is_multi_byte ) | |
| 105 except Exception, e: | |
| 106 stop_err( str( e ) ) | |
| 107 info = dict( type = 'dataset', | |
| 108 dataset_id = data_dict[ 'dataset_id' ], | |
| 109 ext = ext) | |
| 110 | |
| 111 json_file.write( "%s\n" % dumps( info ) ) | |
| 112 | |
| 113 if __name__ == "__main__": __main__() |