# HG changeset patch # User greg # Date 1312480593 14400 # Node ID 2615b9ede2c444c957fdc83b95bd684428c6bde6 # Parent 08ab17610888eef7d9758a1752af0a73d43a6587 Uploaded 1filtering.tar diff -r 08ab17610888 -r 2615b9ede2c4 filtering.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/filtering.py Thu Aug 04 13:56:33 2011 -0400 @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# This tool takes a tab-delimited text file as input and creates filters on columns based on certain properties. +# The tool will skip over invalid lines within the file, informing the user about the number of lines skipped. + +from __future__ import division +import sys, re, os.path +from galaxy import eggs + +# Older py compatibility +try: + set() +except: + from sets import Set as set + +assert sys.version_info[:2] >= ( 2, 4 ) + +def get_operands( filter_condition ): + # Note that the order of all_operators is important + items_to_strip = ['+', '-', '**', '*', '//', '/', '%', '<<', '>>', '&', '|', '^', '~', '<=', '<', '>=', '>', '==', '!=', '<>', ' and ', ' or ', ' not ', ' is ', ' is not ', ' in ', ' not in '] + for item in items_to_strip: + if filter_condition.find( item ) >= 0: + filter_condition = filter_condition.replace( item, ' ' ) + operands = set( filter_condition.split( ' ' ) ) + return operands + +def stop_err( msg ): + sys.stderr.write( msg ) + sys.exit() + +in_fname = sys.argv[1] +out_fname = sys.argv[2] +cond_text = sys.argv[3] +try: + in_columns = int( sys.argv[4] ) + assert sys.argv[5] #check to see that the column types varaible isn't null + in_column_types = sys.argv[5].split( ',' ) +except: + stop_err( "Data does not appear to be tabular. This tool can only be used with tab-delimited data." ) + +# Unescape if input has been escaped +mapped_str = { + '__lt__': '<', + '__le__': '<=', + '__eq__': '==', + '__ne__': '!=', + '__gt__': '>', + '__ge__': '>=', + '__sq__': '\'', + '__dq__': '"', +} +for key, value in mapped_str.items(): + cond_text = cond_text.replace( key, value ) + +# Attempt to determine if the condition includes executable stuff and, if so, exit +secured = dir() +operands = get_operands(cond_text) +for operand in operands: + try: + check = int( operand ) + except: + if operand in secured: + stop_err( "Illegal value '%s' in condition '%s'" % ( operand, cond_text ) ) + +# Prepare the column variable names and wrappers for column data types +cols, type_casts = [], [] +for col in range( 1, in_columns + 1 ): + col_name = "c%d" % col + cols.append( col_name ) + col_type = in_column_types[ col - 1 ] + type_cast = "%s(%s)" % ( col_type, col_name ) + type_casts.append( type_cast ) + +col_str = ', '.join( cols ) # 'c1, c2, c3, c4' +type_cast_str = ', '.join( type_casts ) # 'str(c1), int(c2), int(c3), str(c4)' +assign = "%s = line.split( '\\t' )" % col_str +wrap = "%s = %s" % ( col_str, type_cast_str ) +skipped_lines = 0 +first_invalid_line = 0 +invalid_line = None +lines_kept = 0 +total_lines = 0 +out = open( out_fname, 'wt' ) + +# Read and filter input file, skipping invalid lines +code = ''' +for i, line in enumerate( file( in_fname ) ): + total_lines += 1 + line = line.rstrip( '\\r\\n' ) + if not line or line.startswith( '#' ): + skipped_lines += 1 + if not invalid_line: + first_invalid_line = i + 1 + invalid_line = line + continue + try: + %s + %s + if %s: + lines_kept += 1 + print >> out, line + except: + skipped_lines += 1 + if not invalid_line: + first_invalid_line = i + 1 + invalid_line = line +''' % ( assign, wrap, cond_text ) + +valid_filter = True +try: + exec code +except Exception, e: + out.close() + if str( e ).startswith( 'invalid syntax' ): + valid_filter = False + stop_err( 'Filter condition "%s" likely invalid. See tool tips, syntax and examples.' % cond_text ) + else: + stop_err( str( e ) ) + +if valid_filter: + out.close() + valid_lines = total_lines - skipped_lines + print 'Filtering with %s, ' % cond_text + if valid_lines > 0: + print 'kept %4.2f%% of %d lines.' % ( 100.0*lines_kept/valid_lines, total_lines ) + else: + print 'Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' % cond_text + if skipped_lines > 0: + print 'Skipped %d invalid lines starting at line #%d: "%s"' % ( skipped_lines, first_invalid_line, invalid_line ) diff -r 08ab17610888 -r 2615b9ede2c4 filtering.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/filtering.xml Thu Aug 04 13:56:33 2011 -0400 @@ -0,0 +1,65 @@ + + data on any column using simple expressions + + filtering.py $input $out_file1 "$cond" ${input.metadata.columns} "${input.metadata.column_types}" + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: warningmark + +Double equal signs, ==, must be used as *"equal to"* (e.g., **c1 == 'chr22'**) + +.. class:: infomark + +**TIP:** Attempting to apply a filtering condition may throw exceptions if the data type (e.g., string, integer) in every line of the columns being filtered is not appropriate for the condition (e.g., attempting certain numerical calculations on strings). If an exception is thrown when applying the condition to a line, that line is skipped as invalid for the filter condition. The number of invalid skipped lines is documented in the resulting history item as a "Condition/data issue". + +.. class:: infomark + +**TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* + +----- + +**Syntax** + +The filter tool allows you to restrict the dataset using simple conditional statements. + +- Columns are referenced with **c** and a **number**. For example, **c1** refers to the first column of a tab-delimited file +- Make sure that multi-character operators contain no white space ( e.g., **<=** is valid while **< =** is not valid ) +- When using 'equal-to' operator **double equal sign '==' must be used** ( e.g., **c1=='chr1'** ) +- Non-numerical values must be included in single or double quotes ( e.g., **c6=='+'** ) +- Filtering condition can include logical operators, but **make sure operators are all lower case** ( e.g., **(c1!='chrX' and c1!='chrY') or not c6=='+'** ) + +----- + +**Example** + +- **c1=='chr1'** selects lines in which the first column is chr1 +- **c3-c2<100*c4** selects lines where subtracting column 3 from column 2 is less than the value of column 4 times 100 +- **len(c2.split(',')) < 4** will select lines where the second column has less than four comma separated elements +- **c2>=1** selects lines in which the value of column 2 is greater than or equal to 1 +- Numbers should not contain commas - **c2<=44,554,350** will not work, but **c2<=44554350** will +- Some words in the data can be used, but must be single or double quoted ( e.g., **c3=='exon'** ) + + +