Mercurial > repos > iuc > data_manager_pangolearn

annotate data_manager/pangolearn_dm.py @ 1:f7f2e94d68fc draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
author iuc
date Wed, 02 Jun 2021 16:05:59 +0000
parents e5c31cab174c
children 009409f81fa0
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
1 #!/usr/bin/env python
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
2
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
3 import argparse
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
4 import datetime
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
5 import json
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
6 import operator
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
7 import os
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
8 import shutil
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
9 import sys
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
10 import tarfile
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
11
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
12 import requests
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
13
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
14
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
15 def extract_date(tag_str):
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
16 parts = tag_str.split("_")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
17 assert len(parts) < 3, "expected maximum of two parts, got " + str(parts)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
18 tag_date = datetime.datetime.strptime(parts[0], "%Y-%m-%d")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
19 if len(parts) == 2:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
20 version = int(parts[1])
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
21 assert (
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
22 version < 24 * 60
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
23 ) # because the code stores versions as minutes of the day, it can't handle versions > 1440
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
24 tag_date += datetime.timedelta(minutes=version)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
25 return tag_date
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
26
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
27
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
28 def get_model_list(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
29 existing_release_tags,
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
30 url="https://api.github.com/repos/cov-lineages/pangoLEARN/releases",
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
31 ):
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
32 response = requests.get(url)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
33 if response.status_code == 200:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
34 release_list = json.loads(response.text)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
35 release_info = [
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
36 dict(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
37 tag_name=e["tag_name"],
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
38 name=e["name"],
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
39 date=extract_date(e["tag_name"]),
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
40 tarball_url=e["tarball_url"],
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
41 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
42 for e in release_list
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
43 if e["tag_name"] not in existing_release_tags
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
44 ]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
45 return release_info
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
46 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
47 response.raise_for_status()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
48
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
49
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
50 def filter_by_date(existing_release_tags, start_date=None, end_date=None):
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
51 release_list = get_model_list(existing_release_tags)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
52 return [
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
53 element
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
54 for element in release_list
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
55 if not (
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
56 (end_date is not None and element["date"] > end_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
57 or (start_date is not None and element["date"] < start_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
58 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
59 ]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
60
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
61
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
62 def download_and_unpack(url, output_directory):
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
63 response = requests.get(url)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
64 if response.status_code == 200:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
65 tmp_filename = url.split("/")[-1]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
66 tmpfile = open(tmp_filename, "wb")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
67 tmpfile.write(response.content)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
68 tmpfile.close()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
69 shutil.copy(tmp_filename, "/tmp")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
70 tf = tarfile.open(tmp_filename)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
71 pl_path = tf.next().name
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
72 tf.extractall(output_directory)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
73 os.unlink(tmp_filename)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
74 os.rename(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
75 output_directory + "/" + pl_path + "/" + "pangoLEARN",
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
76 output_directory + "/" + tmp_filename,
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
77 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
78 shutil.rmtree(output_directory + "/" + pl_path)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
79 return tmp_filename
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
80 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
81 response.raise_for_status()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
82
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
83
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
84 def parse_date(d):
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
85 return datetime.datetime.strptime(d, "%Y-%m-%d")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
86
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
87
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
88 if __name__ == "__main__":
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
89
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
90 parser = argparse.ArgumentParser()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
91 parser.add_argument("--testmode", default=False, action="store_true")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
92 parser.add_argument("--latest", default=False, action="store_true")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
93 parser.add_argument("--start_date", type=parse_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
94 parser.add_argument("--end_date", type=parse_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
95 parser.add_argument("--overwrite", default=False, action="store_true")
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
96 parser.add_argument('--pangolearn_format_version')
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
97 parser.add_argument("datatable_name")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
98 parser.add_argument("galaxy_datamanager_filename")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
99 args = parser.parse_args()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
100
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
101 if args.testmode:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
102 releases = filter_by_date(start_date=args.start_date, end_date=args.end_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
103 for release in releases:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
104 print(release["tag_name"], release["tarball_url"].split("/")[-1])
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
105 sys.exit(0)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
106
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
107 with open(args.galaxy_datamanager_filename) as fh:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
108 config = json.load(fh)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
109
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
110 output_directory = config.get("output_data", [{}])[0].get("extra_files_path", None)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
111 data_manager_dict = {}
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
112 data_manager_dict["data_tables"] = config.get("data_tables", {})
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
113 data_manager_dict["data_tables"][args.datatable_name] = data_manager_dict[
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
114 "data_tables"
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
115 ].get(args.datatable_name, [])
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
116
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
117 # NOTE: the data_manager_dict["data_tables"][args.datatable_name] is not actually populated with the
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
118 # contents of the existing data table, so the "no-overwrite" logic and the
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
119 # only-download-what-we-don't-have logic does not in fact work. It is left but unused for now.
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
120 if not args.overwrite:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
121 existing_release_tags = set(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
122 [
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
123 el["value"]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
124 for el in data_manager_dict["data_tables"][args.datatable_name]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
125 ]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
126 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
127 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
128 existing_release_tags = set()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
129 if args.latest:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
130 releases = [get_model_list(existing_release_tags)[0]]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
131 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
132 releases = filter_by_date(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
133 existing_release_tags, start_date=args.start_date, end_date=args.end_date
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
134 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
135 releases_to_download = [
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
136 release
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
137 for release in releases
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
138 if release["tag_name"] not in existing_release_tags
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
139 ]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
140 for release in releases_to_download:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
141 tag = download_and_unpack(release["tarball_url"], output_directory)
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
142 release_date = parse_date(tag)
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
143 if args.pangolearn_format_version is not None:
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
144 version = args.pangolearn_format_version
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
145 else:
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
146 # 2021-05-27 was the first release of pangoLEARN for pangolin 3, which changed DB format
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
147 if release_date >= datetime.datetime(2021, 5, 27):
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
148 version = '3.0'
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
149 else:
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
150 version = '1.0'
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
151 data_manager_dict["data_tables"][args.datatable_name].append(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
152 dict(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
153 value=tag,
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
154 description=release["name"],
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
155 format_version=version,
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
156 path=output_directory + "/" + tag,
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
157 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
158 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
159 data_manager_dict["data_tables"][args.datatable_name].sort(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
160 key=operator.itemgetter("value"), reverse=True
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
161 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
162 with open(args.galaxy_datamanager_filename, "w") as fh:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
163 json.dump(data_manager_dict, fh, indent=2, sort_keys=True)