Mercurial > repos > drosofff > fetch_fasta_from_ncbi
annotate retrieve_fasta_from_NCBI.py @ 13:639daa4c3c1a draft
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
| author | drosofff |
|---|---|
| date | Mon, 15 May 2017 03:10:35 -0400 |
| parents | 0bec3cba5c56 |
| children | 54941746784b |
| rev | line source |
|---|---|
| 0 | 1 #!/usr/bin/env python |
| 2 # -*- coding: utf-8 -*- | |
| 3 """ | |
| 4 From a taxonomy ID retrieves all the nucleotide sequences | |
| 5 It returns a multiFASTA nuc/prot file | |
| 6 | |
| 7 Entrez Database UID common name E-utility Database Name | |
| 8 Nucleotide GI number nuccore | |
| 9 Protein GI number protein | |
| 10 | |
| 11 Retrieve strategy: | |
| 12 | |
| 13 esearch to get total number of UIDs (count) | |
| 14 esearch to get UIDs in batches | |
| 15 loop untile end of UIDs list: | |
| 16 epost to put a batch of UIDs in the history server | |
| 17 efetch to retrieve info from previous post | |
| 18 | |
| 19 retmax of efetch is 1/10 of declared value from NCBI | |
| 20 | |
| 21 queries are 1 sec delayed, to satisfy NCBI guidelines (more than what they request) | |
| 22 | |
| 23 | |
| 24 """ | |
| 2 | 25 import sys |
| 0 | 26 import logging |
| 27 import optparse | |
| 28 import time | |
| 29 import urllib | |
| 30 import urllib2 | |
|
7
cd7de2d6c716
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
5
diff
changeset
|
31 import httplib |
| 0 | 32 import re |
|
12
0bec3cba5c56
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit b6de14061c479f0418cd89e26d6f5ac26e565a07
drosofff
parents:
10
diff
changeset
|
33 |
|
0bec3cba5c56
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit b6de14061c479f0418cd89e26d6f5ac26e565a07
drosofff
parents:
10
diff
changeset
|
34 |
| 0 | 35 class Eutils: |
| 36 | |
| 37 def __init__(self, options, logger): | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
38 """ |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
39 Initialize retrieval parameters |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
40 """ |
| 0 | 41 self.logger = logger |
|
12
0bec3cba5c56
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit b6de14061c479f0418cd89e26d6f5ac26e565a07
drosofff
parents:
10
diff
changeset
|
42 self.base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/" |
| 0 | 43 self.query_string = options.query_string |
| 44 self.dbname = options.dbname | |
| 45 if options.outname: | |
| 46 self.outname = options.outname | |
| 47 else: | |
| 48 self.outname = 'NCBI_download' + '.' + self.dbname + '.fasta' | |
| 49 self.ids = [] | |
| 50 self.retmax_esearch = 100000 | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
51 self.retmax_efetch = 500 |
| 0 | 52 self.count = 0 |
| 53 self.webenv = "" | |
| 54 self.query_key = "" | |
| 55 | |
| 56 def retrieve(self): | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
57 """ |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
58 Retrieve the fasta sequences corresponding to the query |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
59 """ |
| 0 | 60 self.get_count_value() |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
61 |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
62 # If no UIDs are found exit script |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
63 if self.count > 0: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
64 self.get_uids_list() |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
65 self.get_sequences() |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
66 else: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
67 self.logger.info("No UIDs were found. Exiting script.") |
| 0 | 68 |
| 69 def get_count_value(self): | |
| 70 """ | |
| 71 just to retrieve Count (number of UIDs) | |
| 72 Total number of UIDs from the retrieved set to be shown in the XML | |
| 73 output (default=20). By default, ESearch only includes the first 20 | |
| 74 UIDs retrieved in the XML output. If usehistory is set to 'y', | |
| 75 the remainder of the retrieved set will be stored on the History server; | |
| 76 | |
| 77 http://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch | |
| 78 """ | |
| 79 self.logger.info("retrieving data from %s" % self.base) | |
| 80 self.logger.info("for Query: %s and database: %s" % | |
| 81 (self.query_string, self.dbname)) | |
| 82 querylog = self.esearch(self.dbname, self.query_string, '', '', "count") | |
| 83 self.logger.debug("Query response:") | |
| 84 for line in querylog: | |
| 85 self.logger.debug(line.rstrip()) | |
| 86 if '</Count>' in line: | |
| 87 self.count = int(line[line.find('<Count>')+len('<Count>') : line.find('</Count>')]) | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
88 self.logger.info("Found %d UIDs" % self.count) |
| 0 | 89 |
| 90 def get_uids_list(self): | |
| 91 """ | |
| 92 Increasing retmax allows more of the retrieved UIDs to be included in the XML output, | |
| 93 up to a maximum of 100,000 records. | |
| 94 from http://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.ESearch | |
| 95 """ | |
| 96 retmax = self.retmax_esearch | |
| 97 if (self.count > retmax): | |
| 98 num_batches = (self.count / retmax) + 1 | |
| 99 else: | |
| 100 num_batches = 1 | |
| 101 self.logger.info("Batch size for esearch action: %d UIDs" % retmax) | |
| 102 self.logger.info("Number of batches for esearch action: %d " % num_batches) | |
| 103 for n in range(num_batches): | |
| 104 querylog = self.esearch(self.dbname, self.query_string, n*retmax, retmax, '') | |
| 105 for line in querylog: | |
| 106 if '<Id>' in line and '</Id>' in line: | |
| 107 uid = (line[line.find('<Id>')+len('<Id>') : line.find('</Id>')]) | |
| 108 self.ids.append(uid) | |
| 109 self.logger.info("Retrieved %d UIDs" % len(self.ids)) | |
| 110 | |
| 111 def esearch(self, db, term, retstart, retmax, rettype): | |
| 112 url = self.base + "esearch.fcgi" | |
| 113 self.logger.debug("url: %s" % url) | |
| 114 values = {'db': db, | |
| 115 'term': term, | |
| 116 'rettype': rettype, | |
| 117 'retstart': retstart, | |
| 118 'retmax': retmax} | |
| 119 data = urllib.urlencode(values) | |
| 120 self.logger.debug("data: %s" % str(data)) | |
| 121 req = urllib2.Request(url, data) | |
| 122 response = urllib2.urlopen(req) | |
| 123 querylog = response.readlines() | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
124 response.close() |
| 0 | 125 time.sleep(1) |
| 126 return querylog | |
| 127 | |
| 128 def epost(self, db, ids): | |
| 129 url = self.base + "epost.fcgi" | |
| 130 self.logger.debug("url_epost: %s" % url) | |
| 131 values = {'db': db, | |
| 132 'id': ids} | |
| 133 data = urllib.urlencode(values) | |
| 134 req = urllib2.Request(url, data) | |
| 2 | 135 serverResponse = False |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
136 nb_trials = 0 |
| 2 | 137 while not serverResponse: |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
138 nb_trials += 1 |
| 2 | 139 try: |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
140 self.logger.debug("Try number %s for opening and readin URL %s" % ( nb_trials, url+data )) |
| 2 | 141 response = urllib2.urlopen(req) |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
142 querylog = response.readlines() |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
143 response.close() |
| 2 | 144 serverResponse = True |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
145 except urllib2.HTTPError as e: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
146 self.logger.info("urlopen error:%s, %s" % (e.code, e.read() ) ) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
147 self.logger.info("Retrying in 1 sec") |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
148 serverResponse = False |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
149 time.sleep(1) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
150 except urllib2.URLError as e: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
151 self.logger.info("urlopen error: Failed to reach a server") |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
152 self.logger.info("Reason :%s" % ( e.reason ) ) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
153 self.logger.info("Retrying in 1 sec") |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
154 serverResponse = False |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
155 time.sleep(1) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
156 except httplib.IncompleteRead as e: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
157 self.logger.info("IncompleteRead error: %s" % ( e.partial ) ) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
158 self.logger.info("Retrying in 1 sec") |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
159 serverResponse = False |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
160 time.sleep(1) |
| 0 | 161 self.logger.debug("query response:") |
| 162 for line in querylog: | |
| 163 self.logger.debug(line.rstrip()) | |
| 164 if '</QueryKey>' in line: | |
| 165 self.query_key = str(line[line.find('<QueryKey>')+len('<QueryKey>'):line.find('</QueryKey>')]) | |
| 166 if '</WebEnv>' in line: | |
| 167 self.webenv = str(line[line.find('<WebEnv>')+len('<WebEnv>'):line.find('</WebEnv>')]) | |
| 168 self.logger.debug("*** epost action ***") | |
| 169 self.logger.debug("query_key: %s" % self.query_key) | |
| 170 self.logger.debug("webenv: %s" % self.webenv) | |
| 171 time.sleep(1) | |
| 172 | |
| 173 def efetch(self, db, query_key, webenv): | |
| 174 url = self.base + "efetch.fcgi" | |
| 175 self.logger.debug("url_efetch: %s" % url) | |
| 176 values = {'db': db, | |
| 177 'query_key': query_key, | |
| 178 'webenv': webenv, | |
| 179 'rettype': "fasta", | |
| 180 'retmode': "text"} | |
| 181 data = urllib.urlencode(values) | |
| 182 req = urllib2.Request(url, data) | |
| 183 self.logger.debug("data: %s" % str(data)) | |
|
8
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
184 serverTransaction = False |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
185 counter = 0 |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
186 response_code = 0 |
|
8
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
187 while not serverTransaction: |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
188 counter += 1 |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
189 self.logger.info("Server Transaction Trial: %s" % ( counter ) ) |
| 2 | 190 try: |
| 191 response = urllib2.urlopen(req) | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
192 response_code = response.getcode() |
|
8
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
193 fasta = response.read() |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
194 response.close() |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
195 if ( (response_code != 200) or ("Resource temporarily unavailable" in fasta) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
196 or ("Error" in fasta) or (not fasta.startswith(">") ) ): |
|
8
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
197 serverTransaction = False |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
198 else: |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
199 serverTransaction = True |
| 2 | 200 except urllib2.HTTPError as e: |
|
8
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
201 serverTransaction = False |
| 2 | 202 self.logger.info("urlopen error:%s, %s" % (e.code, e.read() ) ) |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
203 except urllib2.URLError as e: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
204 serverTransaction = False |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
205 self.logger.info("urlopen error: Failed to reach a server") |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
206 self.logger.info("Reason :%s" % ( e.reason ) ) |
|
8
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
207 except httplib.IncompleteRead as e: |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
208 serverTransaction = False |
|
cc43a7a11324
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
7
diff
changeset
|
209 self.logger.info("IncompleteRead error: %s" % ( e.partial ) ) |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
210 fasta = self.sanitiser(self.dbname, fasta) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
211 time.sleep(0.1) |
| 0 | 212 return fasta |
| 213 | |
| 214 def sanitiser(self, db, fastaseq): | |
|
5
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
215 if db not in "nuccore protein" : return fastaseq |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
216 regex = re.compile(r"[ACDEFGHIKLMNPQRSTVWYBZ]{49,}") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
217 sane_seqlist = [] |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
218 seqlist = fastaseq.split("\n\n") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
219 for seq in seqlist[:-1]: |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
220 fastalines = seq.split("\n") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
221 if len(fastalines) < 2: |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
222 self.logger.info("Empty sequence for %s" % ("|".join(fastalines[0].split("|")[:4]) ) ) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
223 self.logger.info("%s download is skipped" % ("|".join(fastalines[0].split("|")[:4]) ) ) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
224 continue |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
225 if db == "nuccore": |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
226 badnuc = 0 |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
227 for nucleotide in fastalines[1]: |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
228 if nucleotide not in "ATGC": |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
229 badnuc += 1 |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
230 if float(badnuc)/len(fastalines[1]) > 0.4: |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
231 self.logger.info("%s ambiguous nucleotides in %s or download interrupted at this offset | %s" % ( float(badnuc)/len(fastalines[1]), "|".join(fastalines[0].split("|")[:4]), fastalines[1]) ) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
232 self.logger.info("%s download is skipped" % (fastalines[0].split("|")[:4]) ) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
233 continue |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
234 fastalines[0] = fastalines[0].replace(" ","_")[:100] # remove spaces and trim the header to 100 chars |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
235 cleanseq = "\n".join(fastalines) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
236 sane_seqlist.append(cleanseq) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
237 elif db == "protein": |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
238 fastalines[0] = fastalines[0][0:100] |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
239 fastalines[0] = fastalines[0].replace(" ", "_") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
240 fastalines[0] = fastalines[0].replace("[", "_") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
241 fastalines[0] = fastalines[0].replace("]", "_") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
242 fastalines[0] = fastalines[0].replace("=", "_") |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
243 fastalines[0] = fastalines[0].rstrip("_") # because blast makedb doesn't like it |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
244 fastalines[0] = re.sub(regex, "_", fastalines[0]) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
245 cleanseq = "\n".join(fastalines) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
246 sane_seqlist.append(cleanseq) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
247 self.logger.info("clean sequences appended: %d" % (len(sane_seqlist) ) ) |
|
4ff395248db4
planemo upload for repository https://bitbucket.org/drosofff/gedtools/
drosofff
parents:
2
diff
changeset
|
248 return "\n".join(sane_seqlist) |
| 0 | 249 |
| 250 def get_sequences(self): | |
| 251 """ | |
| 252 Total number of records from the input set to be retrieved, up to a maximum | |
| 253 of 10,000. Optionally, for a large set the value of retstart can be iterated | |
| 254 while holding retmax constant, thereby downloading the entire set in batches | |
| 255 of size retmax. | |
| 256 | |
| 257 http://www.ncbi.nlm.nih.gov/books/NBK25499/#chapter4.EFetch | |
| 258 | |
| 259 """ | |
| 260 batch_size = self.retmax_efetch | |
| 261 count = self.count | |
| 262 uids_list = self.ids | |
| 263 self.logger.info("Batch size for efetch action: %d" % batch_size) | |
| 264 self.logger.info("Number of batches for efetch action: %d" % ((count / batch_size) + 1)) | |
| 265 with open(self.outname, 'w') as out: | |
| 266 for start in range(0, count, batch_size): | |
| 267 end = min(count, start+batch_size) | |
| 268 batch = uids_list[start:end] | |
|
13
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
269 if self.epost(self.dbname, ",".join(batch)) != -1: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
270 mfasta = '' |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
271 while not mfasta: |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
272 self.logger.info("retrieving batch %d" % ((start / batch_size) + 1)) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
273 mfasta = self.efetch(self.dbname, self.query_key, self.webenv) |
|
639daa4c3c1a
planemo upload for repository https://github.com/ARTbio/tools-artbio/tree/master/tools/fetch_fasta_from_ncbi commit ca132a4b5d5d7175e6e8bd62cc518397d14dad17
drosofff
parents:
12
diff
changeset
|
274 out.write(mfasta + '\n') |
| 0 | 275 |
| 276 | |
| 277 LOG_FORMAT = '%(asctime)s|%(levelname)-8s|%(message)s' | |
| 278 LOG_DATEFMT = '%Y-%m-%d %H:%M:%S' | |
| 279 LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] | |
| 280 | |
| 281 | |
| 282 def __main__(): | |
| 283 """ main function """ | |
| 284 parser = optparse.OptionParser(description='Retrieve data from NCBI') | |
| 285 parser.add_option('-i', dest='query_string', help='NCBI Query String') | |
| 286 parser.add_option('-o', dest='outname', help='output file name') | |
| 287 parser.add_option('-l', '--logfile', help='log file (default=stderr)') | |
| 288 parser.add_option('--loglevel', choices=LOG_LEVELS, default='INFO', help='logging level (default: INFO)') | |
| 289 parser.add_option('-d', dest='dbname', help='database type') | |
| 290 (options, args) = parser.parse_args() | |
| 291 if len(args) > 0: | |
| 292 parser.error('Wrong number of arguments') | |
| 293 | |
| 294 log_level = getattr(logging, options.loglevel) | |
| 295 kwargs = {'format': LOG_FORMAT, | |
| 296 'datefmt': LOG_DATEFMT, | |
| 297 'level': log_level} | |
| 298 if options.logfile: | |
| 299 kwargs['filename'] = options.logfile | |
| 300 logging.basicConfig(**kwargs) | |
| 301 logger = logging.getLogger('data_from_NCBI') | |
| 302 | |
| 303 E = Eutils(options, logger) | |
| 304 E.retrieve() | |
| 305 | |
| 306 | |
| 307 if __name__ == "__main__": | |
| 308 __main__() |