proteore_data_manager: data_manager/resource

comparison data_manager/resource_building.py @ 62:add6aa698fb0 draft

"planemo upload commit 4747fc3ca8e24e0f6c0cfcde0992780c6d4ef4ff-dirty"

author	proteore
date	Tue, 09 Jun 2020 16:14:33 +0000
parents	da9e74d3c40d
children	54089754ba12

comparison

equal deleted inserted replaced

-:5504538d24f6
+:add6aa698fb0
 ftp.cwd(ftp_dir)
 ftp.retrbinary("RETR " + file, open(path, 'wb').write)
 ftp.quit()
 return (path)
-def download_from_nextprot_ftp(file,target_directory):
+def download_from_nextprot_ftp(file,target_directory) :
 ftp_dir = "pub/current_release/ac_lists/"
 path = os.path.join(target_directory, file)
 ftp = ftplib.FTP("ftp.nextprot.org")
 ftp.login("anonymous", "anonymous")
 ftp.cwd(ftp_dir)
 ftp.retrbinary("RETR " + file, open(path, 'wb').write)
 ftp.quit()
-return path
+return (path)
-def id_list_from_nextprot_ftp(file) :
+def id_list_from_nextprot_ftp(file,target_directory) :
+ftp_dir = "pub/current_release/ac_lists/"
+path = os.path.join(target_directory, file)
 ftp = ftplib.FTP("ftp.nextprot.org")
 ftp.login("anonymous", "anonymous")
-r = StringIO()
+ftp.cwd(ftp_dir)
-ftp.retrlines("RETR " + file, lambda line: r.write(line + '\n'))
+ftp.retrbinary("RETR " + file, open(path, 'wb').write)
 ftp.quit()
-r.seek(0)
+with open(path,'r') as nextprot_ids :
-ids = r.readlines()
+nextprot_ids = nextprot_ids.read().splitlines()
-ids = [id.strip('\n') for id in ids]
+return (nextprot_ids)
-return (ids)
 #return '' if there's no value in a dictionary, avoid error
 def access_dictionary (dico,key1,key2) :
 if key1 in dico :
 if key2 in dico[key1] :
 _add_data_table_entry(data_manager_dict, data_table_entry, "proteore_"+interactome+"_dictionaries")
 #######################################################################################################
 # 5. nextprot (add protein features)
 #######################################################################################################
 def Build_nextprot_ref_file(data_manager_dict,target_directory):
+nextprot_ids_file = "nextprot_ac_list_all.txt"
-from requests_futures.sessions import FuturesSession
+ids = id_list_from_nextprot_ftp(nextprot_ids_file,target_directory)
-from concurrent.futures import ProcessPoolExecutor
-#Get nextprot ids list
-ids = id_list_from_nextprot_ftp("pub/current_release/ac_lists/nextprot_ac_list_all.txt")
 output_file = 'nextprot_ref_'+ time.strftime("%d-%m-%Y") + ".tsv"
 path = os.path.join(target_directory,output_file)
 name = "neXtProt release "+time.strftime("%d-%m-%Y")
 release_id = "nextprot_ref_"+time.strftime("%d-%m-%Y")
-#open output file to write
 output = open(path, 'w')
 writer = csv.writer(output,delimiter="\t")
-writer.writerow(["NextprotID","MW","SeqLength","IsoPoint","Chr","SubcellLocations","Diseases","TMDomains","ProteinExistence"])
+nextprot_file=[["NextprotID","MW","SeqLength","IsoPoint","Chr","SubcellLocations","Diseases","TMDomains","ProteinExistence"]]
-subset=100
+writer.writerows(nextprot_file)
-ids_subsets = [ids[x:x+subset] for x in range(0, len(ids), subset)]
+for id in ids :
-for ids_subset in ids_subsets:
+query="https://api.nextprot.org/entry/"+id+".json"
+try:
-#Open concurent sessions
+resp = requests.get(url=query)
-with FuturesSession(executor=ProcessPoolExecutor(max_workers=8)) as session:
+except :
-futures = [session.get("https://api.nextprot.org/entry/"+id+".json") for id in ids_subset]
+print ("wainting 1 hour before trying again")
+time.sleep(3600)
-for id,future in zip(ids_subset,futures) :
+resp = requests.get(url=query)
+data = resp.json()
-#Get json dictionary
-try:
+#get info from json dictionary
-res = future.result()
+mass_mol = data["entry"]["isoforms"][0]["massAsString"]
-except:
+seq_length = data['entry']["isoforms"][0]["sequenceLength"]
-print ("sleep 1 hour")
+iso_elec_point = data['entry']["isoforms"][0]["isoelectricPointAsString"]
-time.sleep(3600)
+chr_loc = data['entry']["chromosomalLocations"][0]["chromosome"]
-res = future.result()
+protein_existence = "PE"+str(data['entry']["overview"]['proteinExistence']['level'])
-data = res.json()
+#put all subcell loc in a set
-#get info from json dictionary
+if "subcellular-location" in data['entry']["annotationsByCategory"].keys() :
-mass_mol = data["entry"]["isoforms"][0]["massAsString"]
+subcell_locs = data['entry']["annotationsByCategory"]["subcellular-location"]
-seq_length = data['entry']["isoforms"][0]["sequenceLength"]
+all_subcell_locs = set()
-iso_elec_point = data['entry']["isoforms"][0]["isoelectricPointAsString"]
+for loc in subcell_locs :
-chr_loc = data['entry']["chromosomalLocations"][0]["chromosome"]
+all_subcell_locs.add(loc['cvTermName'])
-protein_existence = "PE"+str(data['entry']["overview"]['proteinExistence']['level'])
+all_subcell_locs.discard("")
+all_subcell_locs = ";".join(all_subcell_locs)
-#put all subcell loc in a set
+else :
-if "subcellular-location" in data['entry']["annotationsByCategory"].keys() :
+all_subcell_locs = "NA"
-subcell_locs = data['entry']["annotationsByCategory"]["subcellular-location"]
-all_subcell_locs = set()
+#put all subcell loc in a set
-for loc in subcell_locs :
+if ('disease') in data['entry']['annotationsByCategory'].keys() :
-all_subcell_locs.add(loc['cvTermName'])
+diseases = data['entry']['annotationsByCategory']['disease']
-all_subcell_locs.discard("")
+all_diseases = set()
-all_subcell_locs = ";".join(all_subcell_locs)
+for disease in diseases :
-else :
+if (disease['cvTermName'] is not None and disease['cvTermName'] != ""):
-all_subcell_locs = "NA"
+all_diseases.add(disease['cvTermName'])
+if len(all_diseases) > 0 : all_diseases = ";".join(all_diseases)
-#put all subcell loc in a set
+else : all_diseases="NA"
-if ('disease') in data['entry']['annotationsByCategory'].keys() :
+else :
-diseases = data['entry']['annotationsByCategory']['disease']
+all_diseases="NA"
-all_diseases = set()
-for disease in diseases :
+#get all tm domain
-if (disease['cvTermName'] is not None and disease['cvTermName'] != ""):
+nb_domains = 0
-all_diseases.add(disease['cvTermName'])
+if  "transmembrane-region" in data['entry']['annotationsByCategory'].keys():
-if len(all_diseases) > 0 : all_diseases = ";".join(all_diseases)
+tm_domains = data['entry']['annotationsByCategory']["transmembrane-region"]
-else : all_diseases="NA"
+all_tm_domains = set()
-else :
+for tm in tm_domains :
-all_diseases="NA"
+all_tm_domains.add(tm['cvTermName'])
+nb_domains+=1
-#get all tm domain
+#print "nb domains ++"
-nb_domains = 0
+#print (nb_domains)
-if  "transmembrane-region" in data['entry']['annotationsByCategory'].keys():
+nextprot_file[:] = []
-tm_domains = data['entry']['annotationsByCategory']["transmembrane-region"]
+nextprot_file.append([id,mass_mol,str(seq_length),iso_elec_point,chr_loc,all_subcell_locs,all_diseases,str(nb_domains),protein_existence])
-all_tm_domains = set()
+writer.writerows(nextprot_file)
-for tm in tm_domains :
-all_tm_domains.add(tm['cvTermName'])
+id = str(10000000000 - int(time.strftime("%Y%m%d")))
-nb_domains+=1
-writer.writerow([id,mass_mol,str(seq_length),iso_elec_point,chr_loc,all_subcell_locs,all_diseases,str(nb_domains),protein_existence])
-id = str(10000000000 - int(time.strftime("%Y%m%d")))
 data_table_entry = dict(id=id, release=release_id, name = name, value = path)
 _add_data_table_entry(data_manager_dict, data_table_entry, "proteore_nextprot_ref")
 #######################################################################################################

Mercurial > repos > proteore > proteore_data_manager

comparison data_manager/resource_building.py @ 62:add6aa698fb0 draft