# HG changeset patch
# User davidvanzessen
# Date 1427716697 14400
# Node ID 072167754619f8569fdea078ac1ac8c94dc85fde
# Parent 2bb24ebbbc38d78bb1e87724d698c1f40ff60c8c
Uploaded
diff -r 2bb24ebbbc38 -r 072167754619 uploadzip.py
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/uploadzip.py Mon Mar 30 07:58:17 2015 -0400
@@ -0,0 +1,391 @@
+#!/usr/bin/env python
+#Processes uploads from the user.
+
+# WARNING: Changes in this tool (particularly as related to parsing) may need
+# to be reflected in galaxy.web.controllers.tool_runner and galaxy.tools
+
+import urllib, sys, os, gzip, tempfile, shutil, re, gzip, zipfile, codecs, binascii
+from galaxy import eggs
+# need to import model before sniff to resolve a circular import dependency
+import galaxy.model
+from galaxy.datatypes.checkers import *
+from galaxy.datatypes import sniff
+from galaxy.datatypes.binary import *
+from galaxy.datatypes.images import Pdf
+from galaxy.datatypes.registry import Registry
+from galaxy import util
+from galaxy.datatypes.util.image_util import *
+from galaxy.util.json import *
+
+try:
+ import Image as PIL
+except ImportError:
+ try:
+ from PIL import Image as PIL
+ except:
+ PIL = None
+
+try:
+ import bz2
+except:
+ bz2 = None
+
+assert sys.version_info[:2] >= ( 2, 4 )
+
+def stop_err( msg, ret=1 ):
+ sys.stderr.write( msg )
+ sys.exit( ret )
+def file_err( msg, dataset, json_file ):
+ json_file.write( to_json_string( dict( type = 'dataset',
+ ext = 'data',
+ dataset_id = dataset.dataset_id,
+ stderr = msg ) ) + "\n" )
+ # never remove a server-side upload
+ if dataset.type in ( 'server_dir', 'path_paste' ):
+ return
+ try:
+ os.remove( dataset.path )
+ except:
+ pass
+def safe_dict(d):
+ """
+ Recursively clone json structure with UTF-8 dictionary keys
+ http://mellowmachines.com/blog/2009/06/exploding-dictionary-with-unicode-keys-as-python-arguments/
+ """
+ if isinstance(d, dict):
+ return dict([(k.encode('utf-8'), safe_dict(v)) for k,v in d.iteritems()])
+ elif isinstance(d, list):
+ return [safe_dict(x) for x in d]
+ else:
+ return d
+def parse_outputs( args ):
+ rval = {}
+ for arg in args:
+ id, files_path, path = arg.split( ':', 2 )
+ rval[int( id )] = ( path, files_path )
+ return rval
+def add_file( dataset, registry, json_file, output_path ):
+ data_type = None
+ line_count = None
+ converted_path = None
+ stdout = None
+ link_data_only = dataset.get( 'link_data_only', 'copy_files' )
+ in_place = dataset.get( 'in_place', True )
+
+ try:
+ ext = dataset.file_type
+ except AttributeError:
+ file_err( 'Unable to process uploaded file, missing file_type parameter.', dataset, json_file )
+ return
+
+ if dataset.type == 'url':
+ try:
+ page = urllib.urlopen( dataset.path ) #page will be .close()ed by sniff methods
+ temp_name, dataset.is_multi_byte = sniff.stream_to_file( page, prefix='url_paste', source_encoding=util.get_charset_from_http_headers( page.headers ) )
+ except Exception, e:
+ file_err( 'Unable to fetch %s\n%s' % ( dataset.path, str( e ) ), dataset, json_file )
+ return
+ dataset.path = temp_name
+ # See if we have an empty file
+ if not os.path.exists( dataset.path ):
+ file_err( 'Uploaded temporary file (%s) does not exist.' % dataset.path, dataset, json_file )
+ return
+ if not os.path.getsize( dataset.path ) > 0:
+ file_err( 'The uploaded file is empty', dataset, json_file )
+ return
+ if not dataset.type == 'url':
+ # Already set is_multi_byte above if type == 'url'
+ try:
+ dataset.is_multi_byte = util.is_multi_byte( codecs.open( dataset.path, 'r', 'utf-8' ).read( 100 ) )
+ except UnicodeDecodeError, e:
+ dataset.is_multi_byte = False
+ # Is dataset an image?
+ image = check_image( dataset.path )
+ if image:
+ if not PIL:
+ image = None
+ # get_image_ext() returns None if nor a supported Image type
+ ext = get_image_ext( dataset.path, image )
+ data_type = ext
+ # Is dataset content multi-byte?
+ elif dataset.is_multi_byte:
+ data_type = 'multi-byte char'
+ ext = sniff.guess_ext( dataset.path, is_multi_byte=True )
+ # Is dataset content supported sniffable binary?
+ else:
+ type_info = Binary.is_sniffable_binary( dataset.path )
+ if type_info:
+ data_type = type_info[0]
+ ext = type_info[1]
+ data_type="binary"
+ if not data_type:
+ # See if we have a gzipped file, which, if it passes our restrictions, we'll uncompress
+ is_gzipped, is_valid = check_gzip( dataset.path )
+ if is_gzipped and not is_valid:
+ file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file )
+ return
+ elif is_gzipped and is_valid:
+ if link_data_only == 'copy_files':
+ # We need to uncompress the temp_name file, but BAM files must remain compressed in the BGZF format
+ CHUNK_SIZE = 2**20 # 1Mb
+ fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_gunzip_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False )
+ gzipped_file = gzip.GzipFile( dataset.path, 'rb' )
+ while 1:
+ try:
+ chunk = gzipped_file.read( CHUNK_SIZE )
+ except IOError:
+ os.close( fd )
+ os.remove( uncompressed )
+ file_err( 'Problem decompressing gzipped data', dataset, json_file )
+ return
+ if not chunk:
+ break
+ os.write( fd, chunk )
+ os.close( fd )
+ gzipped_file.close()
+ # Replace the gzipped file with the decompressed file if it's safe to do so
+ if dataset.type in ( 'server_dir', 'path_paste' ) or not in_place:
+ dataset.path = uncompressed
+ else:
+ shutil.move( uncompressed, dataset.path )
+ os.chmod(dataset.path, 0644)
+ dataset.name = dataset.name.rstrip( '.gz' )
+ data_type = 'gzip'
+ if not data_type and bz2 is not None:
+ # See if we have a bz2 file, much like gzip
+ is_bzipped, is_valid = check_bz2( dataset.path )
+ if is_bzipped and not is_valid:
+ file_err( 'The gzipped uploaded file contains inappropriate content', dataset, json_file )
+ return
+ elif is_bzipped and is_valid:
+ if link_data_only == 'copy_files':
+ # We need to uncompress the temp_name file
+ CHUNK_SIZE = 2**20 # 1Mb
+ fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_bunzip2_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False )
+ bzipped_file = bz2.BZ2File( dataset.path, 'rb' )
+ while 1:
+ try:
+ chunk = bzipped_file.read( CHUNK_SIZE )
+ except IOError:
+ os.close( fd )
+ os.remove( uncompressed )
+ file_err( 'Problem decompressing bz2 compressed data', dataset, json_file )
+ return
+ if not chunk:
+ break
+ os.write( fd, chunk )
+ os.close( fd )
+ bzipped_file.close()
+ # Replace the bzipped file with the decompressed file if it's safe to do so
+ if dataset.type in ( 'server_dir', 'path_paste' ) or not in_place:
+ dataset.path = uncompressed
+ else:
+ shutil.move( uncompressed, dataset.path )
+ os.chmod(dataset.path, 0644)
+ dataset.name = dataset.name.rstrip( '.bz2' )
+ data_type = 'bz2'
+ if not data_type:
+ # See if we have a zip archive
+ is_zipped = check_zip( dataset.path )
+ if is_zipped:
+ if link_data_only == 'copy_files':
+ CHUNK_SIZE = 2**20 # 1Mb
+ uncompressed = None
+ uncompressed_name = None
+ unzipped = False
+ z = zipfile.ZipFile( dataset.path )
+ for name in z.namelist():
+ if name.endswith('/'):
+ continue
+ if unzipped:
+ stdout = 'ZIP file contained more than one file, only the first file was added to Galaxy.'
+ break
+ fd, uncompressed = tempfile.mkstemp( prefix='data_id_%s_upload_zip_' % dataset.dataset_id, dir=os.path.dirname( output_path ), text=False )
+ if sys.version_info[:2] >= ( 2, 6 ):
+ zipped_file = z.open( name )
+ while 1:
+ try:
+ chunk = zipped_file.read( CHUNK_SIZE )
+ except IOError:
+ os.close( fd )
+ os.remove( uncompressed )
+ file_err( 'Problem decompressing zipped data', dataset, json_file )
+ return
+ if not chunk:
+ break
+ os.write( fd, chunk )
+ os.close( fd )
+ zipped_file.close()
+ uncompressed_name = name
+ unzipped = True
+ else:
+ # python < 2.5 doesn't have a way to read members in chunks(!)
+ try:
+ outfile = open( uncompressed, 'wb' )
+ outfile.write( z.read( name ) )
+ outfile.close()
+ uncompressed_name = name
+ unzipped = True
+ except IOError:
+ os.close( fd )
+ os.remove( uncompressed )
+ file_err( 'Problem decompressing zipped data', dataset, json_file )
+ return
+ z.close()
+ # Replace the zipped file with the decompressed file if it's safe to do so
+ if uncompressed is not None:
+ if dataset.type in ( 'server_dir', 'path_paste' ) or not in_place:
+ dataset.path = uncompressed
+ else:
+ shutil.move( uncompressed, dataset.path )
+ os.chmod(dataset.path, 0644)
+ dataset.name = uncompressed_name
+ data_type = 'zip'
+ if not data_type:
+ if check_binary( dataset.path ):
+ # We have a binary dataset, but it is not Bam, Sff or Pdf
+ data_type = 'binary'
+ #binary_ok = False
+ parts = dataset.name.split( "." )
+ if len( parts ) > 1:
+ ext = parts[-1].strip().lower()
+ if not Binary.is_ext_unsniffable(ext):
+ file_err( 'The uploaded binary file contains inappropriate content', dataset, json_file )
+ return
+ elif Binary.is_ext_unsniffable(ext) and dataset.file_type != ext:
+ err_msg = "You must manually set the 'File Format' to '%s' when uploading %s files." % ( ext.capitalize(), ext )
+ file_err( err_msg, dataset, json_file )
+ return
+ if not data_type:
+ # We must have a text file
+ if check_html( dataset.path ):
+ file_err( 'The uploaded file contains inappropriate HTML content', dataset, json_file )
+ return
+ if data_type != 'binary':
+ if link_data_only == 'copy_files':
+ if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]:
+ in_place = False
+ # Convert universal line endings to Posix line endings, but allow the user to turn it off,
+ # so that is becomes possible to upload gzip, bz2 or zip files with binary data without
+ # corrupting the content of those files.
+ if dataset.to_posix_lines:
+ if dataset.space_to_tab:
+ line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
+ else:
+ line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
+ if dataset.file_type == 'auto':
+ ext = sniff.guess_ext( dataset.path, registry.sniff_order )
+ else:
+ ext = dataset.file_type
+ data_type = ext
+ # Save job info for the framework
+ if ext == 'auto' and dataset.ext:
+ ext = dataset.ext
+ if ext == 'auto':
+ ext = 'data'
+ datatype = registry.get_datatype_by_extension( ext )
+ if dataset.type in ( 'server_dir', 'path_paste' ) and link_data_only == 'link_to_files':
+ # Never alter a file that will not be copied to Galaxy's local file store.
+ if datatype.dataset_content_needs_grooming( dataset.path ):
+ err_msg = 'The uploaded files need grooming, so change your Copy data into Galaxy? selection to be ' + \
+ 'Copy files into Galaxy instead of Link to files without copying into Galaxy so grooming can be performed.'
+ file_err( err_msg, dataset, json_file )
+ return
+ if link_data_only == 'copy_files' and dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]:
+ # Move the dataset to its "real" path
+ if converted_path is not None:
+ shutil.copy( converted_path, output_path )
+ try:
+ os.remove( converted_path )
+ except:
+ pass
+ else:
+ # This should not happen, but it's here just in case
+ shutil.copy( dataset.path, output_path )
+ elif link_data_only == 'copy_files':
+ shutil.move( dataset.path, output_path )
+ # Write the job info
+ stdout = stdout or 'uploaded %s file' % data_type
+ info = dict( type = 'dataset',
+ dataset_id = dataset.dataset_id,
+ ext = ext,
+ stdout = stdout,
+ name = dataset.name,
+ line_count = line_count )
+ if dataset.get('uuid', None) is not None:
+ info['uuid'] = dataset.get('uuid')
+ json_file.write( to_json_string( info ) + "\n" )
+
+ if link_data_only == 'copy_files' and datatype.dataset_content_needs_grooming( output_path ):
+ # Groom the dataset content if necessary
+ datatype.groom_dataset_content( output_path )
+
+def add_composite_file( dataset, registry, json_file, output_path, files_path ):
+ if dataset.composite_files:
+ os.mkdir( files_path )
+ for name, value in dataset.composite_files.iteritems():
+ value = util.bunch.Bunch( **value )
+ if dataset.composite_file_paths[ value.name ] is None and not value.optional:
+ file_err( 'A required composite data file was not provided (%s)' % name, dataset, json_file )
+ break
+ elif dataset.composite_file_paths[value.name] is not None:
+ dp = dataset.composite_file_paths[value.name][ 'path' ]
+ isurl = dp.find('://') <> -1 # todo fixme
+ if isurl:
+ try:
+ temp_name, dataset.is_multi_byte = sniff.stream_to_file( urllib.urlopen( dp ), prefix='url_paste' )
+ except Exception, e:
+ file_err( 'Unable to fetch %s\n%s' % ( dp, str( e ) ), dataset, json_file )
+ return
+ dataset.path = temp_name
+ dp = temp_name
+ if not value.is_binary:
+ if dataset.composite_file_paths[ value.name ].get( 'space_to_tab', value.space_to_tab ):
+ sniff.convert_newlines_sep2tabs( dp )
+ else:
+ sniff.convert_newlines( dp )
+ shutil.move( dp, os.path.join( files_path, name ) )
+ # Move the dataset to its "real" path
+ shutil.move( dataset.primary_file, output_path )
+ # Write the job info
+ info = dict( type = 'dataset',
+ dataset_id = dataset.dataset_id,
+ stdout = 'uploaded %s file' % dataset.file_type )
+ json_file.write( to_json_string( info ) + "\n" )
+
+def __main__():
+
+ if len( sys.argv ) < 4:
+ print >>sys.stderr, 'usage: upload.py