|
0
|
1 #!/usr/bin/env python3
|
|
|
2 """
|
|
|
3 Parses tab delimited database files
|
|
|
4 """
|
|
|
5 # Info
|
|
|
6 __author__ = 'Jason Anthony Vander Heiden'
|
|
|
7 from changeo import __version__, __date__
|
|
|
8
|
|
|
9 # Imports
|
|
|
10 import csv
|
|
|
11 import os
|
|
|
12 import re
|
|
|
13 from argparse import ArgumentParser
|
|
|
14 from collections import OrderedDict
|
|
|
15
|
|
|
16 from textwrap import dedent
|
|
|
17 from time import time
|
|
|
18 from Bio import SeqIO
|
|
|
19 from Bio.Seq import Seq
|
|
|
20 from Bio.SeqRecord import SeqRecord
|
|
|
21 from Bio.Alphabet import IUPAC
|
|
|
22
|
|
|
23 # Presto and changeo imports
|
|
|
24 from presto.Defaults import default_delimiter, default_out_args
|
|
|
25 from presto.Annotation import flattenAnnotation
|
|
|
26 from presto.IO import getOutputHandle, printLog, printProgress, printMessage
|
|
|
27 from changeo.Commandline import CommonHelpFormatter, getCommonArgParser, parseCommonArgs
|
|
|
28 from changeo.IO import getDbWriter, readDbFile, countDbFile
|
|
|
29
|
|
|
30 # Defaults
|
|
|
31 default_id_field = 'SEQUENCE_ID'
|
|
|
32 default_seq_field = 'SEQUENCE_IMGT'
|
|
|
33 default_germ_field = 'GERMLINE_IMGT_D_MASK'
|
|
|
34 default_index_field = 'INDEX'
|
|
|
35
|
|
|
36 # TODO: convert SQL-ish operations to modify_func() as per ParseHeaders
|
|
|
37
|
|
|
38 def getDbSeqRecord(db_record, id_field, seq_field, meta_fields=None,
|
|
|
39 delimiter=default_delimiter):
|
|
|
40 """
|
|
|
41 Parses a database record into a SeqRecord
|
|
|
42
|
|
|
43 Arguments:
|
|
|
44 db_record = a dictionary containing a database record
|
|
|
45 id_field = the field containing identifiers
|
|
|
46 seq_field = the field containing sequences
|
|
|
47 meta_fields = a list of fields to add to sequence annotations
|
|
|
48 delimiter = a tuple of delimiters for (fields, values, value lists)
|
|
|
49
|
|
|
50 Returns:
|
|
|
51 a SeqRecord
|
|
|
52 """
|
|
|
53 # Return None if ID or sequence fields are empty
|
|
|
54 if not db_record[id_field] or not db_record[seq_field]:
|
|
|
55 return None
|
|
|
56
|
|
|
57 # Create description string
|
|
|
58 desc_dict = OrderedDict([('ID', db_record[id_field])])
|
|
|
59 if meta_fields is not None:
|
|
|
60 desc_dict.update([(f, db_record[f]) for f in meta_fields if f in db_record])
|
|
|
61 desc_str = flattenAnnotation(desc_dict, delimiter=delimiter)
|
|
|
62
|
|
|
63 # Create SeqRecord
|
|
|
64 seq_record = SeqRecord(Seq(db_record[seq_field], IUPAC.ambiguous_dna),
|
|
|
65 id=desc_str, name=desc_str, description='')
|
|
|
66
|
|
|
67 return seq_record
|
|
|
68
|
|
|
69
|
|
|
70 def splitDbFile(db_file, field, num_split=None, out_args=default_out_args):
|
|
|
71 """
|
|
|
72 Divides a tab-delimited database file into segments by description tags
|
|
|
73
|
|
|
74 Arguments:
|
|
|
75 db_file = filename of the tab-delimited database file to split
|
|
|
76 field = the field name by which to split db_file
|
|
|
77 num_split = the numerical threshold by which to group sequences;
|
|
|
78 if None treat field as textual
|
|
|
79 out_args = common output argument dictionary from parseCommonArgs
|
|
|
80
|
|
|
81 Returns:
|
|
|
82 a list of output file names
|
|
|
83 """
|
|
|
84 log = OrderedDict()
|
|
|
85 log['START'] = 'ParseDb'
|
|
|
86 log['COMMAND'] = 'split'
|
|
|
87 log['FILE'] = os.path.basename(db_file)
|
|
|
88 log['FIELD'] = field
|
|
|
89 log['NUM_SPLIT'] = num_split
|
|
|
90 printLog(log)
|
|
|
91
|
|
|
92 # Open IgRecord reader iter object
|
|
|
93 reader = readDbFile(db_file, ig=False)
|
|
|
94
|
|
|
95 # Determine total numbers of records
|
|
|
96 rec_count = countDbFile(db_file)
|
|
|
97
|
|
|
98 start_time = time()
|
|
|
99 count = 0
|
|
|
100 # Sort records into files based on textual field
|
|
|
101 if num_split is None:
|
|
|
102 # Create set of unique field tags
|
|
|
103 tmp_iter = readDbFile(db_file, ig=False)
|
|
|
104 tag_list = list(set([row[field] for row in tmp_iter]))
|
|
|
105
|
|
|
106 # Forbidden characters in filename and replacements
|
|
|
107 noGood = {'\/':'f','\\':'b','?':'q','\%':'p','*':'s',':':'c',
|
|
|
108 '\|':'pi','\"':'dq','\'':'sq','<':'gt','>':'lt',' ':'_'}
|
|
|
109 # Replace forbidden characters in tag_list
|
|
|
110 tag_dict = {}
|
|
|
111 for tag in tag_list:
|
|
|
112 for c,r in noGood.items():
|
|
|
113 tag_dict[tag] = (tag_dict.get(tag, tag).replace(c,r) \
|
|
|
114 if c in tag else tag_dict.get(tag, tag))
|
|
|
115
|
|
|
116 # Create output handles
|
|
|
117 handles_dict = {tag:getOutputHandle(db_file,
|
|
|
118 '%s-%s' % (field, label),
|
|
|
119 out_type = out_args['out_type'],
|
|
|
120 out_name = out_args['out_name'],
|
|
|
121 out_dir = out_args['out_dir'])
|
|
|
122 for tag, label in tag_dict.items()}
|
|
|
123
|
|
|
124 # Create Db writer instances
|
|
|
125 writers_dict = {tag:getDbWriter(handles_dict[tag], db_file)
|
|
|
126 for tag in tag_dict}
|
|
|
127
|
|
|
128 # Iterate over IgRecords
|
|
|
129 for row in reader:
|
|
|
130 printProgress(count, rec_count, 0.05, start_time)
|
|
|
131 count += 1
|
|
|
132 # Write row to appropriate file
|
|
|
133 tag = row[field]
|
|
|
134 writers_dict[tag].writerow(row)
|
|
|
135
|
|
|
136 # Sort records into files based on numeric num_split
|
|
|
137 else:
|
|
|
138 num_split = float(num_split)
|
|
|
139
|
|
|
140 # Create output handles
|
|
|
141 handles_dict = {'under':getOutputHandle(db_file,
|
|
|
142 'under-%.1f' % num_split,
|
|
|
143 out_type = out_args['out_type'],
|
|
|
144 out_name = out_args['out_name'],
|
|
|
145 out_dir = out_args['out_dir']),
|
|
|
146 'atleast':getOutputHandle(db_file,
|
|
|
147 'atleast-%.1f' % num_split,
|
|
|
148 out_type = out_args['out_type'],
|
|
|
149 out_name = out_args['out_name'],
|
|
|
150 out_dir = out_args['out_dir'])}
|
|
|
151
|
|
|
152 # Create Db writer instances
|
|
|
153 writers_dict = {'under':getDbWriter(handles_dict['under'], db_file),
|
|
|
154 'atleast':getDbWriter(handles_dict['atleast'], db_file)}
|
|
|
155
|
|
|
156 # Iterate over IgRecords
|
|
|
157 for row in reader:
|
|
|
158 printProgress(count, rec_count, 0.05, start_time)
|
|
|
159 count += 1
|
|
|
160 tag = row[field]
|
|
|
161 tag = 'under' if float(tag) < num_split else 'atleast'
|
|
|
162 writers_dict[tag].writerow(row)
|
|
|
163
|
|
|
164 # Write log
|
|
|
165 printProgress(count, rec_count, 0.05, start_time)
|
|
|
166 log = OrderedDict()
|
|
|
167 for i, k in enumerate(handles_dict):
|
|
|
168 log['OUTPUT%i' % (i + 1)] = os.path.basename(handles_dict[k].name)
|
|
|
169 log['RECORDS'] = rec_count
|
|
|
170 log['PARTS'] = len(handles_dict)
|
|
|
171 log['END'] = 'ParseDb'
|
|
|
172 printLog(log)
|
|
|
173
|
|
|
174 # Close output file handles
|
|
|
175 for t in handles_dict: handles_dict[t].close()
|
|
|
176
|
|
|
177 return [handles_dict[t].name for t in handles_dict]
|
|
|
178
|
|
|
179
|
|
|
180 # TODO: SHOULD ALLOW FOR UNSORTED CLUSTER COLUMN
|
|
|
181 # TODO: SHOULD ALLOW FOR GROUPING FIELDS
|
|
|
182 def convertDbClip(db_file, id_field=default_id_field, seq_field=default_seq_field,
|
|
|
183 germ_field=default_germ_field, cluster_field=None,
|
|
|
184 meta_fields=None, out_args=default_out_args):
|
|
|
185 """
|
|
|
186 Builds fasta files from database records
|
|
|
187
|
|
|
188 Arguments:
|
|
|
189 db_file = the database file name
|
|
|
190 id_field = the field containing identifiers
|
|
|
191 seq_field = the field containing sample sequences
|
|
|
192 germ_field = the field containing germline sequences
|
|
|
193 cluster_field = the field containing clonal groupings
|
|
|
194 if None write the germline for each record
|
|
|
195 meta_fields = a list of fields to add to sequence annotations
|
|
|
196 out_args = common output argument dictionary from parseCommonArgs
|
|
|
197
|
|
|
198 Returns:
|
|
|
199 the output file name
|
|
|
200 """
|
|
|
201 log = OrderedDict()
|
|
|
202 log['START'] = 'ParseDb'
|
|
|
203 log['COMMAND'] = 'fasta'
|
|
|
204 log['FILE'] = os.path.basename(db_file)
|
|
|
205 log['ID_FIELD'] = id_field
|
|
|
206 log['SEQ_FIELD'] = seq_field
|
|
|
207 log['GERM_FIELD'] = germ_field
|
|
|
208 log['CLUSTER_FIELD'] = cluster_field
|
|
|
209 if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
|
|
|
210 printLog(log)
|
|
|
211
|
|
|
212 # Open file handles
|
|
|
213 db_iter = readDbFile(db_file, ig=False)
|
|
|
214 pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
|
|
|
215 out_name=out_args['out_name'], out_type='clip')
|
|
|
216 # Count records
|
|
|
217 result_count = countDbFile(db_file)
|
|
|
218
|
|
|
219 # Iterate over records
|
|
|
220 start_time = time()
|
|
|
221 rec_count = germ_count = pass_count = fail_count = 0
|
|
|
222 cluster_last = None
|
|
|
223 for rec in db_iter:
|
|
|
224 # Print progress for previous iteration
|
|
|
225 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
226 rec_count += 1
|
|
|
227
|
|
|
228 # Update cluster ID
|
|
|
229 cluster = rec.get(cluster_field, None)
|
|
|
230
|
|
|
231 # Get germline SeqRecord when needed
|
|
|
232 if cluster_field is None:
|
|
|
233 germ = getDbSeqRecord(rec, id_field, germ_field, meta_fields,
|
|
|
234 delimiter=out_args['delimiter'])
|
|
|
235 germ.id = '>' + germ.id
|
|
|
236 elif cluster != cluster_last:
|
|
|
237 germ = getDbSeqRecord(rec, cluster_field, germ_field,
|
|
|
238 delimiter=out_args['delimiter'])
|
|
|
239 germ.id = '>' + germ.id
|
|
|
240 else:
|
|
|
241 germ = None
|
|
|
242
|
|
|
243 # Get read SeqRecord
|
|
|
244 seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields,
|
|
|
245 delimiter=out_args['delimiter'])
|
|
|
246
|
|
|
247 # Write germline
|
|
|
248 if germ is not None:
|
|
|
249 germ_count += 1
|
|
|
250 SeqIO.write(germ, pass_handle, 'fasta')
|
|
|
251
|
|
|
252 # Write sequences
|
|
|
253 if seq is not None:
|
|
|
254 pass_count += 1
|
|
|
255 SeqIO.write(seq, pass_handle, 'fasta')
|
|
|
256 else:
|
|
|
257 fail_count += 1
|
|
|
258
|
|
|
259 # Set last cluster ID
|
|
|
260 cluster_last = cluster
|
|
|
261
|
|
|
262 # Print counts
|
|
|
263 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
264 log = OrderedDict()
|
|
|
265 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
266 log['RECORDS'] = rec_count
|
|
|
267 log['GERMLINES'] = germ_count
|
|
|
268 log['PASS'] = pass_count
|
|
|
269 log['FAIL'] = fail_count
|
|
|
270 log['END'] = 'ParseDb'
|
|
|
271 printLog(log)
|
|
|
272
|
|
|
273 # Close file handles
|
|
|
274 pass_handle.close()
|
|
|
275
|
|
|
276 return pass_handle.name
|
|
|
277
|
|
|
278
|
|
|
279 def convertDbFasta(db_file, id_field=default_id_field, seq_field=default_seq_field,
|
|
|
280 meta_fields=None, out_args=default_out_args):
|
|
|
281 """
|
|
|
282 Builds fasta files from database records
|
|
|
283
|
|
|
284 Arguments:
|
|
|
285 db_file = the database file name
|
|
|
286 id_field = the field containing identifiers
|
|
|
287 seq_field = the field containing sequences
|
|
|
288 meta_fields = a list of fields to add to sequence annotations
|
|
|
289 out_args = common output argument dictionary from parseCommonArgs
|
|
|
290
|
|
|
291 Returns:
|
|
|
292 the output file name
|
|
|
293 """
|
|
|
294 log = OrderedDict()
|
|
|
295 log['START'] = 'ParseDb'
|
|
|
296 log['COMMAND'] = 'fasta'
|
|
|
297 log['FILE'] = os.path.basename(db_file)
|
|
|
298 log['ID_FIELD'] = id_field
|
|
|
299 log['SEQ_FIELD'] = seq_field
|
|
|
300 if meta_fields is not None: log['META_FIELDS'] = ','.join(meta_fields)
|
|
|
301 printLog(log)
|
|
|
302
|
|
|
303 # Open file handles
|
|
|
304 out_type = 'fasta'
|
|
|
305 db_iter = readDbFile(db_file, ig=False)
|
|
|
306 pass_handle = getOutputHandle(db_file, out_label='sequences', out_dir=out_args['out_dir'],
|
|
|
307 out_name=out_args['out_name'], out_type=out_type)
|
|
|
308 # Count records
|
|
|
309 result_count = countDbFile(db_file)
|
|
|
310
|
|
|
311 # Iterate over records
|
|
|
312 start_time = time()
|
|
|
313 rec_count = pass_count = fail_count = 0
|
|
|
314 for rec in db_iter:
|
|
|
315 # Print progress for previous iteration
|
|
|
316 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
317 rec_count += 1
|
|
|
318
|
|
|
319 # Get SeqRecord
|
|
|
320 seq = getDbSeqRecord(rec, id_field, seq_field, meta_fields, out_args['delimiter'])
|
|
|
321
|
|
|
322 # Write sequences
|
|
|
323 if seq is not None:
|
|
|
324 pass_count += 1
|
|
|
325 SeqIO.write(seq, pass_handle, out_type)
|
|
|
326 else:
|
|
|
327 fail_count += 1
|
|
|
328
|
|
|
329 # Print counts
|
|
|
330 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
331 log = OrderedDict()
|
|
|
332 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
333 log['RECORDS'] = rec_count
|
|
|
334 log['PASS'] = pass_count
|
|
|
335 log['FAIL'] = fail_count
|
|
|
336 log['END'] = 'ParseDb'
|
|
|
337 printLog(log)
|
|
|
338
|
|
|
339 # Close file handles
|
|
|
340 pass_handle.close()
|
|
|
341
|
|
|
342 return pass_handle.name
|
|
|
343
|
|
|
344
|
|
|
345 def addDbFile(db_file, fields, values, out_args=default_out_args):
|
|
|
346 """
|
|
|
347 Adds field and value pairs to a database file
|
|
|
348
|
|
|
349 Arguments:
|
|
|
350 db_file = the database file name
|
|
|
351 fields = a list of fields to add
|
|
|
352 values = a list of values to assign to all rows of each field
|
|
|
353 out_args = common output argument dictionary from parseCommonArgs
|
|
|
354
|
|
|
355 Returns:
|
|
|
356 the output file name
|
|
|
357 """
|
|
|
358 log = OrderedDict()
|
|
|
359 log['START'] = 'ParseDb'
|
|
|
360 log['COMMAND'] = 'add'
|
|
|
361 log['FILE'] = os.path.basename(db_file)
|
|
|
362 log['FIELDS'] = ','.join(fields)
|
|
|
363 log['VALUES'] = ','.join(values)
|
|
|
364 printLog(log)
|
|
|
365
|
|
|
366 # Open file handles
|
|
|
367 db_iter = readDbFile(db_file, ig=False)
|
|
|
368 pass_handle = getOutputHandle(db_file, out_label='parse-add', out_dir=out_args['out_dir'],
|
|
|
369 out_name=out_args['out_name'], out_type='tab')
|
|
|
370 pass_writer = getDbWriter(pass_handle, db_file, add_fields=fields)
|
|
|
371 # Count records
|
|
|
372 result_count = countDbFile(db_file)
|
|
|
373
|
|
|
374 # Define fields and values to append
|
|
|
375 add_dict = {k:v for k,v in zip(fields, values) if k not in db_iter.fieldnames}
|
|
|
376
|
|
|
377 # Iterate over records
|
|
|
378 start_time = time()
|
|
|
379 rec_count = 0
|
|
|
380 for rec in db_iter:
|
|
|
381 # Print progress for previous iteration
|
|
|
382 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
383 rec_count += 1
|
|
|
384 # Write updated row
|
|
|
385 rec.update(add_dict)
|
|
|
386 pass_writer.writerow(rec)
|
|
|
387
|
|
|
388 # Print counts
|
|
|
389 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
390 log = OrderedDict()
|
|
|
391 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
392 log['RECORDS'] = rec_count
|
|
|
393 log['END'] = 'ParseDb'
|
|
|
394 printLog(log)
|
|
|
395
|
|
|
396 # Close file handles
|
|
|
397 pass_handle.close()
|
|
|
398
|
|
|
399 return pass_handle.name
|
|
|
400
|
|
|
401
|
|
|
402 def indexDbFile(db_file, field=default_index_field, out_args=default_out_args):
|
|
|
403 """
|
|
|
404 Adds an index column to a database file
|
|
|
405
|
|
|
406 Arguments:
|
|
|
407 db_file = the database file name
|
|
|
408 field = the name of the index field to add
|
|
|
409 out_args = common output argument dictionary from parseCommonArgs
|
|
|
410
|
|
|
411 Returns:
|
|
|
412 the output file name
|
|
|
413 """
|
|
|
414 log = OrderedDict()
|
|
|
415 log['START'] = 'ParseDb'
|
|
|
416 log['COMMAND'] = 'index'
|
|
|
417 log['FILE'] = os.path.basename(db_file)
|
|
|
418 log['FIELD'] = field
|
|
|
419 printLog(log)
|
|
|
420
|
|
|
421 # Open file handles
|
|
|
422 db_iter = readDbFile(db_file, ig=False)
|
|
|
423 pass_handle = getOutputHandle(db_file, out_label='parse-index', out_dir=out_args['out_dir'],
|
|
|
424 out_name=out_args['out_name'], out_type='tab')
|
|
|
425 pass_writer = getDbWriter(pass_handle, db_file, add_fields=field)
|
|
|
426 # Count records
|
|
|
427 result_count = countDbFile(db_file)
|
|
|
428
|
|
|
429 # Iterate over records
|
|
|
430 start_time = time()
|
|
|
431 rec_count = 0
|
|
|
432 for rec in db_iter:
|
|
|
433 # Print progress for previous iteration
|
|
|
434 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
435 rec_count += 1
|
|
|
436
|
|
|
437 # Add count and write updated row
|
|
|
438 rec.update({field:rec_count})
|
|
|
439 pass_writer.writerow(rec)
|
|
|
440
|
|
|
441 # Print counts
|
|
|
442 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
443 log = OrderedDict()
|
|
|
444 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
445 log['RECORDS'] = rec_count
|
|
|
446 log['END'] = 'ParseDb'
|
|
|
447 printLog(log)
|
|
|
448
|
|
|
449 # Close file handles
|
|
|
450 pass_handle.close()
|
|
|
451
|
|
|
452 return pass_handle.name
|
|
|
453
|
|
|
454
|
|
|
455 def dropDbFile(db_file, fields, out_args=default_out_args):
|
|
|
456 """
|
|
|
457 Deletes entire fields from a database file
|
|
|
458
|
|
|
459 Arguments:
|
|
|
460 db_file = the database file name
|
|
|
461 fields = a list of fields to drop
|
|
|
462 out_args = common output argument dictionary from parseCommonArgs
|
|
|
463
|
|
|
464 Returns:
|
|
|
465 the output file name
|
|
|
466 """
|
|
|
467 log = OrderedDict()
|
|
|
468 log['START'] = 'ParseDb'
|
|
|
469 log['COMMAND'] = 'add'
|
|
|
470 log['FILE'] = os.path.basename(db_file)
|
|
|
471 log['FIELDS'] = ','.join(fields)
|
|
|
472 printLog(log)
|
|
|
473
|
|
|
474 # Open file handles
|
|
|
475 db_iter = readDbFile(db_file, ig=False)
|
|
|
476 pass_handle = getOutputHandle(db_file, out_label='parse-drop', out_dir=out_args['out_dir'],
|
|
|
477 out_name=out_args['out_name'], out_type='tab')
|
|
|
478 pass_writer = getDbWriter(pass_handle, db_file, exclude_fields=fields)
|
|
|
479 # Count records
|
|
|
480 result_count = countDbFile(db_file)
|
|
|
481
|
|
|
482 # Iterate over records
|
|
|
483 start_time = time()
|
|
|
484 rec_count = 0
|
|
|
485 for rec in db_iter:
|
|
|
486 # Print progress for previous iteration
|
|
|
487 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
488 rec_count += 1
|
|
|
489 # Write row
|
|
|
490 pass_writer.writerow(rec)
|
|
|
491
|
|
|
492 # Print counts
|
|
|
493 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
494 log = OrderedDict()
|
|
|
495 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
496 log['RECORDS'] = rec_count
|
|
|
497 log['END'] = 'ParseDb'
|
|
|
498 printLog(log)
|
|
|
499
|
|
|
500 # Close file handles
|
|
|
501 pass_handle.close()
|
|
|
502
|
|
|
503 return pass_handle.name
|
|
|
504
|
|
|
505
|
|
|
506 def deleteDbFile(db_file, fields, values, logic='any', regex=False,
|
|
|
507 out_args=default_out_args):
|
|
|
508 """
|
|
|
509 Deletes records from a database file
|
|
|
510
|
|
|
511 Arguments:
|
|
|
512 db_file = the database file name
|
|
|
513 fields = a list of fields to check for deletion criteria
|
|
|
514 values = a list of values defining deletion targets
|
|
|
515 logic = one of 'any' or 'all' defining whether one or all fields must have a match.
|
|
|
516 regex = if False do exact full string matches; if True allow partial regex matches.
|
|
|
517 out_args = common output argument dictionary from parseCommonArgs
|
|
|
518
|
|
|
519 Returns:
|
|
|
520 the output file name
|
|
|
521 """
|
|
|
522 # Define string match function
|
|
|
523 if regex:
|
|
|
524 def _match_func(x, patterns): return any([re.search(p, x) for p in patterns])
|
|
|
525 else:
|
|
|
526 def _match_func(x, patterns): return x in patterns
|
|
|
527
|
|
|
528 # Define logic function
|
|
|
529 if logic == 'any':
|
|
|
530 _logic_func = any
|
|
|
531 elif logic == 'all':
|
|
|
532 _logic_func = all
|
|
|
533
|
|
|
534 log = OrderedDict()
|
|
|
535 log['START'] = 'ParseDb'
|
|
|
536 log['COMMAND'] = 'delete'
|
|
|
537 log['FILE'] = os.path.basename(db_file)
|
|
|
538 log['FIELDS'] = ','.join(fields)
|
|
|
539 log['VALUES'] = ','.join(values)
|
|
|
540 printLog(log)
|
|
|
541
|
|
|
542 # Open file handles
|
|
|
543 db_iter = readDbFile(db_file, ig=False)
|
|
|
544 pass_handle = getOutputHandle(db_file, out_label='parse-delete', out_dir=out_args['out_dir'],
|
|
|
545 out_name=out_args['out_name'], out_type='tab')
|
|
|
546 pass_writer = getDbWriter(pass_handle, db_file)
|
|
|
547 # Count records
|
|
|
548 result_count = countDbFile(db_file)
|
|
|
549
|
|
|
550 # Iterate over records
|
|
|
551 start_time = time()
|
|
|
552 rec_count = pass_count = fail_count = 0
|
|
|
553 for rec in db_iter:
|
|
|
554 # Print progress for previous iteration
|
|
|
555 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
556 rec_count += 1
|
|
|
557
|
|
|
558 # Check for deletion values in all fields
|
|
|
559 delete = _logic_func([_match_func(rec.get(f, False), values) for f in fields])
|
|
|
560
|
|
|
561 # Write sequences
|
|
|
562 if not delete:
|
|
|
563 pass_count += 1
|
|
|
564 pass_writer.writerow(rec)
|
|
|
565 else:
|
|
|
566 fail_count += 1
|
|
|
567
|
|
|
568 # Print counts
|
|
|
569 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
570 log = OrderedDict()
|
|
|
571 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
572 log['RECORDS'] = rec_count
|
|
|
573 log['KEPT'] = pass_count
|
|
|
574 log['DELETED'] = fail_count
|
|
|
575 log['END'] = 'ParseDb'
|
|
|
576 printLog(log)
|
|
|
577
|
|
|
578 # Close file handles
|
|
|
579 pass_handle.close()
|
|
|
580
|
|
|
581 return pass_handle.name
|
|
|
582
|
|
|
583
|
|
|
584 def renameDbFile(db_file, fields, names, out_args=default_out_args):
|
|
|
585 """
|
|
|
586 Renames fields in a database file
|
|
|
587
|
|
|
588 Arguments:
|
|
|
589 db_file = the database file name
|
|
|
590 fields = a list of fields to rename
|
|
|
591 values = a list of new names for fields
|
|
|
592 out_args = common output argument dictionary from parseCommonArgs
|
|
|
593
|
|
|
594 Returns:
|
|
|
595 the output file name
|
|
|
596 """
|
|
|
597 log = OrderedDict()
|
|
|
598 log['START'] = 'ParseDb'
|
|
|
599 log['COMMAND'] = 'rename'
|
|
|
600 log['FILE'] = os.path.basename(db_file)
|
|
|
601 log['FIELDS'] = ','.join(fields)
|
|
|
602 log['NAMES'] = ','.join(names)
|
|
|
603 printLog(log)
|
|
|
604
|
|
|
605 # Open file handles
|
|
|
606 db_iter = readDbFile(db_file, ig=False)
|
|
|
607 pass_handle = getOutputHandle(db_file, out_label='parse-rename', out_dir=out_args['out_dir'],
|
|
|
608 out_name=out_args['out_name'], out_type='tab')
|
|
|
609
|
|
|
610 # Get header and rename fields
|
|
|
611 header = (readDbFile(db_file, ig=False)).fieldnames
|
|
|
612 for f, n in zip(fields, names):
|
|
|
613 i = header.index(f)
|
|
|
614 header[i] = n
|
|
|
615
|
|
|
616 # Open writer and write new header
|
|
|
617 # TODO: should modify getDbWriter to take a list of fields
|
|
|
618 pass_writer = csv.DictWriter(pass_handle, fieldnames=header, dialect='excel-tab')
|
|
|
619 pass_writer.writeheader()
|
|
|
620
|
|
|
621 # Count records
|
|
|
622 result_count = countDbFile(db_file)
|
|
|
623
|
|
|
624 # Iterate over records
|
|
|
625 start_time = time()
|
|
|
626 rec_count = 0
|
|
|
627 for rec in db_iter:
|
|
|
628 # Print progress for previous iteration
|
|
|
629 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
630 rec_count += 1
|
|
|
631 # TODO: repeating renaming is unnecessary. should had a non-dict reader/writer to DbCore
|
|
|
632 # Rename fields
|
|
|
633 for f, n in zip(fields, names):
|
|
|
634 rec[n] = rec.pop(f)
|
|
|
635 # Write
|
|
|
636 pass_writer.writerow(rec)
|
|
|
637
|
|
|
638 # Print counts
|
|
|
639 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
640 log = OrderedDict()
|
|
|
641 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
642 log['RECORDS'] = rec_count
|
|
|
643 log['END'] = 'ParseDb'
|
|
|
644 printLog(log)
|
|
|
645
|
|
|
646 # Close file handles
|
|
|
647 pass_handle.close()
|
|
|
648
|
|
|
649 return pass_handle.name
|
|
|
650
|
|
|
651
|
|
|
652 def selectDbFile(db_file, fields, values, logic='any', regex=False,
|
|
|
653 out_args=default_out_args):
|
|
|
654 """
|
|
|
655 Selects records from a database file
|
|
|
656
|
|
|
657 Arguments:
|
|
|
658 db_file = the database file name
|
|
|
659 fields = a list of fields to check for selection criteria
|
|
|
660 values = a list of values defining selection targets
|
|
|
661 logic = one of 'any' or 'all' defining whether one or all fields must have a match.
|
|
|
662 regex = if False do exact full string matches; if True allow partial regex matches.
|
|
|
663 out_args = common output argument dictionary from parseCommonArgs
|
|
|
664
|
|
|
665 Returns:
|
|
|
666 the output file name
|
|
|
667 """
|
|
|
668 # Define string match function
|
|
|
669 if regex:
|
|
|
670 def _match_func(x, patterns): return any([re.search(p, x) for p in patterns])
|
|
|
671 else:
|
|
|
672 def _match_func(x, patterns): return x in patterns
|
|
|
673
|
|
|
674 # Define logic function
|
|
|
675 if logic == 'any':
|
|
|
676 _logic_func = any
|
|
|
677 elif logic == 'all':
|
|
|
678 _logic_func = all
|
|
|
679
|
|
|
680 # Print console log
|
|
|
681 log = OrderedDict()
|
|
|
682 log['START'] = 'ParseDb'
|
|
|
683 log['COMMAND'] = 'select'
|
|
|
684 log['FILE'] = os.path.basename(db_file)
|
|
|
685 log['FIELDS'] = ','.join(fields)
|
|
|
686 log['VALUES'] = ','.join(values)
|
|
|
687 log['REGEX'] =regex
|
|
|
688 printLog(log)
|
|
|
689
|
|
|
690 # Open file handles
|
|
|
691 db_iter = readDbFile(db_file, ig=False)
|
|
|
692 pass_handle = getOutputHandle(db_file, out_label='parse-select', out_dir=out_args['out_dir'],
|
|
|
693 out_name=out_args['out_name'], out_type='tab')
|
|
|
694 pass_writer = getDbWriter(pass_handle, db_file)
|
|
|
695 # Count records
|
|
|
696 result_count = countDbFile(db_file)
|
|
|
697
|
|
|
698 # Iterate over records
|
|
|
699 start_time = time()
|
|
|
700 rec_count = pass_count = fail_count = 0
|
|
|
701 for rec in db_iter:
|
|
|
702 # Print progress for previous iteration
|
|
|
703 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
704 rec_count += 1
|
|
|
705
|
|
|
706 # Check for selection values in all fields
|
|
|
707 select = _logic_func([_match_func(rec.get(f, False), values) for f in fields])
|
|
|
708
|
|
|
709 # Write sequences
|
|
|
710 if select:
|
|
|
711 pass_count += 1
|
|
|
712 pass_writer.writerow(rec)
|
|
|
713 else:
|
|
|
714 fail_count += 1
|
|
|
715
|
|
|
716 # Print counts
|
|
|
717 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
718 log = OrderedDict()
|
|
|
719 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
720 log['RECORDS'] = rec_count
|
|
|
721 log['SELECTED'] = pass_count
|
|
|
722 log['DISCARDED'] = fail_count
|
|
|
723 log['END'] = 'ParseDb'
|
|
|
724 printLog(log)
|
|
|
725
|
|
|
726 # Close file handles
|
|
|
727 pass_handle.close()
|
|
|
728
|
|
|
729 return pass_handle.name
|
|
|
730
|
|
|
731
|
|
|
732 def sortDbFile(db_file, field, numeric=False, descend=False,
|
|
|
733 out_args=default_out_args):
|
|
|
734 """
|
|
|
735 Sorts records by values in an annotation field
|
|
|
736
|
|
|
737 Arguments:
|
|
|
738 db_file = the database filename
|
|
|
739 field = the field name to sort by
|
|
|
740 numeric = if True sort field numerically;
|
|
|
741 if False sort field alphabetically
|
|
|
742 descend = if True sort in descending order;
|
|
|
743 if False sort in ascending order
|
|
|
744
|
|
|
745 out_args = common output argument dictionary from parseCommonArgs
|
|
|
746
|
|
|
747 Returns:
|
|
|
748 the output file name
|
|
|
749 """
|
|
|
750 log = OrderedDict()
|
|
|
751 log['START'] = 'ParseDb'
|
|
|
752 log['COMMAND'] = 'sort'
|
|
|
753 log['FILE'] = os.path.basename(db_file)
|
|
|
754 log['FIELD'] = field
|
|
|
755 log['NUMERIC'] = numeric
|
|
|
756 printLog(log)
|
|
|
757
|
|
|
758 # Open file handles
|
|
|
759 db_iter = readDbFile(db_file, ig=False)
|
|
|
760 pass_handle = getOutputHandle(db_file, out_label='parse-sort', out_dir=out_args['out_dir'],
|
|
|
761 out_name=out_args['out_name'], out_type='tab')
|
|
|
762 pass_writer = getDbWriter(pass_handle, db_file)
|
|
|
763
|
|
|
764
|
|
|
765 # Store all records in a dictionary
|
|
|
766 start_time = time()
|
|
|
767 printMessage("Indexing: Running", start_time=start_time)
|
|
|
768 db_dict = {i:r for i, r in enumerate(db_iter)}
|
|
|
769 result_count = len(db_dict)
|
|
|
770
|
|
|
771 # Sort db_dict by field values
|
|
|
772 tag_dict = {k:v[field] for k, v in db_dict.items()}
|
|
|
773 if numeric: tag_dict = {k:float(v or 0) for k, v in tag_dict.items()}
|
|
|
774 sorted_keys = sorted(tag_dict, key=tag_dict.get, reverse=descend)
|
|
|
775 printMessage("Indexing: Done", start_time=start_time, end=True)
|
|
|
776
|
|
|
777 # Iterate over records
|
|
|
778 start_time = time()
|
|
|
779 rec_count = 0
|
|
|
780 for key in sorted_keys:
|
|
|
781 # Print progress for previous iteration
|
|
|
782 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
783 rec_count += 1
|
|
|
784
|
|
|
785 # Write records
|
|
|
786 pass_writer.writerow(db_dict[key])
|
|
|
787
|
|
|
788 # Print counts
|
|
|
789 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
790 log = OrderedDict()
|
|
|
791 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
792 log['RECORDS'] = rec_count
|
|
|
793 log['END'] = 'ParseDb'
|
|
|
794 printLog(log)
|
|
|
795
|
|
|
796 # Close file handles
|
|
|
797 pass_handle.close()
|
|
|
798
|
|
|
799 return pass_handle.name
|
|
|
800
|
|
|
801
|
|
|
802 def updateDbFile(db_file, field, values, updates, out_args=default_out_args):
|
|
|
803 """
|
|
|
804 Updates field and value pairs to a database file
|
|
|
805
|
|
|
806 Arguments:
|
|
|
807 db_file = the database file name
|
|
|
808 field = the field to update
|
|
|
809 values = a list of values to specifying which rows to update
|
|
|
810 updates = a list of values to update each value with
|
|
|
811 out_args = common output argument dictionary from parseCommonArgs
|
|
|
812
|
|
|
813 Returns:
|
|
|
814 the output file name
|
|
|
815 """
|
|
|
816 log = OrderedDict()
|
|
|
817 log['START'] = 'ParseDb'
|
|
|
818 log['COMMAND'] = 'update'
|
|
|
819 log['FILE'] = os.path.basename(db_file)
|
|
|
820 log['FIELD'] = field
|
|
|
821 log['VALUES'] = ','.join(values)
|
|
|
822 log['UPDATES'] = ','.join(updates)
|
|
|
823 printLog(log)
|
|
|
824
|
|
|
825 # Open file handles
|
|
|
826 db_iter = readDbFile(db_file, ig=False)
|
|
|
827 pass_handle = getOutputHandle(db_file, out_label='parse-update', out_dir=out_args['out_dir'],
|
|
|
828 out_name=out_args['out_name'], out_type='tab')
|
|
|
829 pass_writer = getDbWriter(pass_handle, db_file)
|
|
|
830 # Count records
|
|
|
831 result_count = countDbFile(db_file)
|
|
|
832
|
|
|
833 # Iterate over records
|
|
|
834 start_time = time()
|
|
|
835 rec_count = pass_count = 0
|
|
|
836 for rec in db_iter:
|
|
|
837 # Print progress for previous iteration
|
|
|
838 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
839 rec_count += 1
|
|
|
840
|
|
|
841 # Updated values if found
|
|
|
842 for x, y in zip(values, updates):
|
|
|
843 if rec[field] == x:
|
|
|
844 rec[field] = y
|
|
|
845 pass_count += 1
|
|
|
846
|
|
|
847 # Write records
|
|
|
848 pass_writer.writerow(rec)
|
|
|
849
|
|
|
850 # Print counts
|
|
|
851 printProgress(rec_count, result_count, 0.05, start_time)
|
|
|
852 log = OrderedDict()
|
|
|
853 log['OUTPUT'] = os.path.basename(pass_handle.name)
|
|
|
854 log['RECORDS'] = rec_count
|
|
|
855 log['UPDATED'] = pass_count
|
|
|
856 log['END'] = 'ParseDb'
|
|
|
857 printLog(log)
|
|
|
858
|
|
|
859 # Close file handles
|
|
|
860 pass_handle.close()
|
|
|
861
|
|
|
862 return pass_handle.name
|
|
|
863
|
|
|
864
|
|
|
865 def getArgParser():
|
|
|
866 """
|
|
|
867 Defines the ArgumentParser
|
|
|
868
|
|
|
869 Arguments:
|
|
|
870 None
|
|
|
871
|
|
|
872 Returns:
|
|
|
873 an ArgumentParser object
|
|
|
874 """
|
|
|
875 # Define input and output field help message
|
|
|
876 fields = dedent(
|
|
|
877 '''
|
|
|
878 output files:
|
|
|
879 sequences
|
|
|
880 FASTA formatted sequences output from the subcommands fasta and clip.
|
|
|
881 <field>-<value>
|
|
|
882 database files partitioned by annotation <field> and <value>.
|
|
|
883 parse-<command>
|
|
|
884 output of the database modification functions where <command> is one of
|
|
|
885 the subcommands add, index, drop, delete, rename, select, sort or update.
|
|
|
886
|
|
|
887 required fields:
|
|
|
888 SEQUENCE_ID
|
|
|
889
|
|
|
890 optional fields:
|
|
|
891 JUNCTION, SEQUENCE_IMGT, SEQUENCE_VDJ, GERMLINE_IMGT, GERMLINE_VDJ,
|
|
|
892 GERMLINE_IMGT_D_MASK, GERMLINE_VDJ_D_MASK,
|
|
|
893 GERMLINE_IMGT_V_REGION, GERMLINE_VDJ_V_REGION
|
|
|
894
|
|
|
895 output fields:
|
|
|
896 None
|
|
|
897 ''')
|
|
|
898
|
|
|
899 # Define ArgumentParser
|
|
|
900 parser = ArgumentParser(description=__doc__, epilog=fields,
|
|
|
901 formatter_class=CommonHelpFormatter)
|
|
|
902 parser.add_argument('--version', action='version',
|
|
|
903 version='%(prog)s:' + ' %s-%s' %(__version__, __date__))
|
|
|
904 subparsers = parser.add_subparsers(title='subcommands', dest='command', metavar='',
|
|
|
905 help='Database operation')
|
|
|
906 # TODO: This is a temporary fix for Python issue 9253
|
|
|
907 subparsers.required = True
|
|
|
908
|
|
|
909 # Define parent parser
|
|
|
910 parser_parent = getCommonArgParser(seq_in=False, seq_out=False, db_in=True,
|
|
|
911 failed=False, log=False)
|
|
|
912
|
|
|
913 # Subparser to convert database entries to sequence file
|
|
|
914 parser_seq = subparsers.add_parser('fasta', parents=[parser_parent],
|
|
|
915 formatter_class=CommonHelpFormatter,
|
|
|
916 help='Creates a fasta file from database records')
|
|
|
917 parser_seq.add_argument('--if', action='store', dest='id_field',
|
|
|
918 default=default_id_field,
|
|
|
919 help='The name of the field containing identifiers')
|
|
|
920 parser_seq.add_argument('--sf', action='store', dest='seq_field',
|
|
|
921 default=default_seq_field,
|
|
|
922 help='The name of the field containing sequences')
|
|
|
923 parser_seq.add_argument('--mf', nargs='+', action='store', dest='meta_fields',
|
|
|
924 help='List of annotation fields to add to the sequence description')
|
|
|
925 parser_seq.set_defaults(func=convertDbFasta)
|
|
|
926
|
|
|
927 # Subparser to convert database entries to clip-fasta file
|
|
|
928 parser_clip = subparsers.add_parser('clip', parents=[parser_parent],
|
|
|
929 formatter_class=CommonHelpFormatter,
|
|
|
930 help='''Creates a clip-fasta file from database
|
|
|
931 records, wherein germline sequences precede
|
|
|
932 each clone and are denoted by ">>" headers.''')
|
|
|
933 parser_clip.add_argument('--if', action='store', dest='id_field',
|
|
|
934 default=default_id_field,
|
|
|
935 help='The name of the field containing identifiers')
|
|
|
936 parser_clip.add_argument('--sf', action='store', dest='seq_field',
|
|
|
937 default=default_seq_field,
|
|
|
938 help='The name of the field containing reads')
|
|
|
939 parser_clip.add_argument('--gf', action='store', dest='germ_field',
|
|
|
940 default=default_germ_field,
|
|
|
941 help='The name of the field containing germline sequences')
|
|
|
942 parser_clip.add_argument('--cf', action='store', dest='cluster_field', default=None,
|
|
|
943 help='The name of the field containing containing sorted clone IDs')
|
|
|
944 parser_clip.add_argument('--mf', nargs='+', action='store', dest='meta_fields',
|
|
|
945 help='List of annotation fields to add to the sequence description')
|
|
|
946 parser_clip.set_defaults(func=convertDbClip)
|
|
|
947
|
|
|
948 # Subparser to partition files by annotation values
|
|
|
949 parser_split = subparsers.add_parser('split', parents=[parser_parent],
|
|
|
950 formatter_class=CommonHelpFormatter,
|
|
|
951 help='Splits database files by field values')
|
|
|
952 parser_split.add_argument('-f', action='store', dest='field', type=str, required=True,
|
|
|
953 help='Annotation field by which to split database files.')
|
|
|
954 parser_split.add_argument('--num', action='store', dest='num_split', type=float, default=None,
|
|
|
955 help='''Specify to define the field as numeric and group
|
|
|
956 records by whether they are less than or at least
|
|
|
957 (greater than or equal to) the specified value.''')
|
|
|
958 parser_split.set_defaults(func=splitDbFile)
|
|
|
959
|
|
|
960 # Subparser to add records
|
|
|
961 parser_add = subparsers.add_parser('add', parents=[parser_parent],
|
|
|
962 formatter_class=CommonHelpFormatter,
|
|
|
963 help='Adds field and value pairs')
|
|
|
964 parser_add.add_argument('-f', nargs='+', action='store', dest='fields', required=True,
|
|
|
965 help='The name of the fields to add.')
|
|
|
966 parser_add.add_argument('-u', nargs='+', action='store', dest='values', required=True,
|
|
|
967 help='The value to assign to all rows for each field.')
|
|
|
968 parser_add.set_defaults(func=addDbFile)
|
|
|
969
|
|
|
970 # Subparser to delete records
|
|
|
971 parser_delete = subparsers.add_parser('delete', parents=[parser_parent],
|
|
|
972 formatter_class=CommonHelpFormatter,
|
|
|
973 help='Deletes specific records')
|
|
|
974 parser_delete.add_argument('-f', nargs='+', action='store', dest='fields', required=True,
|
|
|
975 help='The name of the fields to check for deletion criteria.')
|
|
|
976 parser_delete.add_argument('-u', nargs='+', action='store', dest='values', default=['', 'NA'],
|
|
|
977 help='''The values defining which records to delete. A value
|
|
|
978 may appear in any of the fields specified with -f.''')
|
|
|
979 parser_delete.add_argument('--logic', action='store', dest='logic',
|
|
|
980 choices=('any', 'all'), default='any',
|
|
|
981 help='''Defines whether a value may appear in any field (any)
|
|
|
982 or whether it must appear in all fields (all).''')
|
|
|
983 parser_delete.add_argument('--regex', action='store_true', dest='regex',
|
|
|
984 help='''If specified, treat values as regular expressions
|
|
|
985 and allow partial string matches.''')
|
|
|
986 parser_delete.set_defaults(func=deleteDbFile)
|
|
|
987
|
|
|
988 # Subparser to drop fields
|
|
|
989 parser_drop = subparsers.add_parser('drop', parents=[parser_parent],
|
|
|
990 formatter_class=CommonHelpFormatter,
|
|
|
991 help='Deletes entire fields')
|
|
|
992 parser_drop.add_argument('-f', nargs='+', action='store', dest='fields', required=True,
|
|
|
993 help='The name of the fields to delete from the database.')
|
|
|
994 parser_drop.set_defaults(func=dropDbFile)
|
|
|
995
|
|
|
996 # Subparser to index fields
|
|
|
997 parser_index = subparsers.add_parser('index', parents=[parser_parent],
|
|
|
998 formatter_class=CommonHelpFormatter,
|
|
|
999 help='Adds a numeric index field')
|
|
|
1000 parser_index.add_argument('-f', action='store', dest='field',
|
|
|
1001 default=default_index_field,
|
|
|
1002 help='The name of the index field to add to the database.')
|
|
|
1003 parser_index.set_defaults(func=indexDbFile)
|
|
|
1004
|
|
|
1005 # Subparser to rename fields
|
|
|
1006 parser_rename = subparsers.add_parser('rename', parents=[parser_parent],
|
|
|
1007 formatter_class=CommonHelpFormatter,
|
|
|
1008 help='Renames fields')
|
|
|
1009 parser_rename.add_argument('-f', nargs='+', action='store', dest='fields', required=True,
|
|
|
1010 help='List of fields to rename.')
|
|
|
1011 parser_rename.add_argument('-k', nargs='+', action='store', dest='names', required=True,
|
|
|
1012 help='List of new names for each field.')
|
|
|
1013 parser_rename.set_defaults(func=renameDbFile)
|
|
|
1014
|
|
|
1015 # Subparser to select records
|
|
|
1016 parser_select = subparsers.add_parser('select', parents=[parser_parent],
|
|
|
1017 formatter_class=CommonHelpFormatter,
|
|
|
1018 help='Selects specific records')
|
|
|
1019 parser_select.add_argument('-f', nargs='+', action='store', dest='fields', required=True,
|
|
|
1020 help='The name of the fields to check for selection criteria.')
|
|
|
1021 parser_select.add_argument('-u', nargs='+', action='store', dest='values', required=True,
|
|
|
1022 help='''The values defining with records to select. A value
|
|
|
1023 may appear in any of the fields specified with -f.''')
|
|
|
1024 parser_select.add_argument('--logic', action='store', dest='logic',
|
|
|
1025 choices=('any', 'all'), default='any',
|
|
|
1026 help='''Defines whether a value may appear in any field (any)
|
|
|
1027 or whether it must appear in all fields (all).''')
|
|
|
1028 parser_select.add_argument('--regex', action='store_true', dest='regex',
|
|
|
1029 help='''If specified, treat values as regular expressions
|
|
|
1030 and allow partial string matches.''')
|
|
|
1031 parser_select.set_defaults(func=selectDbFile)
|
|
|
1032
|
|
|
1033 # Subparser to sort file by records
|
|
|
1034 parser_sort = subparsers.add_parser('sort', parents=[parser_parent],
|
|
|
1035 formatter_class=CommonHelpFormatter,
|
|
|
1036 help='Sorts records by field values')
|
|
|
1037 parser_sort.add_argument('-f', action='store', dest='field', type=str, required=True,
|
|
|
1038 help='The annotation field by which to sort records.')
|
|
|
1039 parser_sort.add_argument('--num', action='store_true', dest='numeric', default=False,
|
|
|
1040 help='''Specify to define the sort column as numeric rather
|
|
|
1041 than textual.''')
|
|
|
1042 parser_sort.add_argument('--descend', action='store_true', dest='descend',
|
|
|
1043 help='''If specified, sort records in descending, rather
|
|
|
1044 than ascending, order by values in the target field.''')
|
|
|
1045 parser_sort.set_defaults(func=sortDbFile)
|
|
|
1046
|
|
|
1047 # Subparser to update records
|
|
|
1048 parser_update = subparsers.add_parser('update', parents=[parser_parent],
|
|
|
1049 formatter_class=CommonHelpFormatter,
|
|
|
1050 help='Updates field and value pairs')
|
|
|
1051 parser_update.add_argument('-f', action='store', dest='field', required=True,
|
|
|
1052 help='The name of the field to update.')
|
|
|
1053 parser_update.add_argument('-u', nargs='+', action='store', dest='values', required=True,
|
|
|
1054 help='The values that will be replaced.')
|
|
|
1055 parser_update.add_argument('-t', nargs='+', action='store', dest='updates', required=True,
|
|
|
1056 help='''The new value to assign to each selected row.''')
|
|
|
1057 parser_update.set_defaults(func=updateDbFile)
|
|
|
1058
|
|
|
1059 return parser
|
|
|
1060
|
|
|
1061
|
|
|
1062 if __name__ == '__main__':
|
|
|
1063 """
|
|
|
1064 Parses command line arguments and calls main function
|
|
|
1065 """
|
|
|
1066 # Parse arguments
|
|
|
1067 parser = getArgParser()
|
|
|
1068 args = parser.parse_args()
|
|
|
1069 args_dict = parseCommonArgs(args)
|
|
|
1070 # Convert case of fields
|
|
|
1071 if 'id_field' in args_dict:
|
|
|
1072 args_dict['id_field'] = args_dict['id_field'].upper()
|
|
|
1073 if 'seq_field' in args_dict:
|
|
|
1074 args_dict['seq_field'] = args_dict['seq_field'].upper()
|
|
|
1075 if 'germ_field' in args_dict:
|
|
|
1076 args_dict['germ_field'] = args_dict['germ_field'].upper()
|
|
|
1077 if 'field' in args_dict:
|
|
|
1078 args_dict['field'] = args_dict['field'].upper()
|
|
|
1079 if 'cluster_field' in args_dict and args_dict['cluster_field'] is not None:
|
|
|
1080 args_dict['cluster_field'] = args_dict['cluster_field'].upper()
|
|
|
1081 if 'meta_fields' in args_dict and args_dict['meta_fields'] is not None:
|
|
|
1082 args_dict['meta_fields'] = [f.upper() for f in args_dict['meta_fields']]
|
|
|
1083 if 'fields' in args_dict:
|
|
|
1084 args_dict['fields'] = [f.upper() for f in args_dict['fields']]
|
|
|
1085
|
|
|
1086 # Check modify_args arguments
|
|
|
1087 if args.command == 'add' and len(args_dict['fields']) != len(args_dict['values']):
|
|
|
1088 parser.error('You must specify exactly one value (-u) per field (-f)')
|
|
|
1089 elif args.command == 'rename' and len(args_dict['fields']) != len(args_dict['names']):
|
|
|
1090 parser.error('You must specify exactly one new name (-k) per field (-f)')
|
|
|
1091 elif args.command == 'update' and len(args_dict['values']) != len(args_dict['updates']):
|
|
|
1092 parser.error('You must specify exactly one value (-u) per replacement (-t)')
|
|
|
1093
|
|
|
1094 # Call parser function for each database file
|
|
|
1095 del args_dict['command']
|
|
|
1096 del args_dict['func']
|
|
|
1097 del args_dict['db_files']
|
|
|
1098 for f in args.__dict__['db_files']:
|
|
|
1099 args_dict['db_file'] = f
|
|
|
1100 args.func(**args_dict)
|
|
|
1101 |
