Mercurial > repos > iuc > data_manager_pangolearn

annotate data_manager/pangolearn_dm.py @ 4:4a37568620a1 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
author iuc
date Tue, 05 Apr 2022 18:39:38 +0000
parents 009409f81fa0
children 587aa4f08914
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
1 #!/usr/bin/env python
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
2
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
3 import argparse
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
4 import datetime
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
5 import json
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
6 import operator
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
7 import os
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
8 import shutil
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
9 import sys
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
10 import tarfile
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
11
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
12 import requests
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
13
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
14
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
15 def get_model_list(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
16 existing_release_tags,
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
17 url="https://api.github.com/repos/cov-lineages/pangoLEARN/releases"
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
18 ):
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
19 page_num = 0
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
20 while True:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
21 page_num += 1
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
22 response = requests.get(url + f'?page={page_num}')
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
23 if response.status_code == 200:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
24 release_list_chunk = json.loads(response.text)
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
25 if not release_list_chunk:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
26 # past the last page of results
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
27 return
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
28 for e in release_list_chunk:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
29 if e["tag_name"] in existing_release_tags:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
30 continue
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
31 if e["prerelease"]:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
32 continue
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
33 yield dict(
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
34 tag_name=e["tag_name"],
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
35 name=e["name"],
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
36 date=parse_date(e["tag_name"]),
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
37 tarball_url=e["tarball_url"],
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
38 )
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
39 else:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
40 response.raise_for_status()
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
41
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
42
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
43 def filter_by_date(existing_release_tags, start_date=None, end_date=None):
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
44 ret = []
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
45 for release in get_model_list(existing_release_tags):
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
46 if start_date and release["date"] < start_date:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
47 break
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
48 if not end_date or release["date"] <= end_date:
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
49 ret.append(release)
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
50
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
51 return ret
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
52
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
53
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
54 def download_and_unpack(url, output_directory):
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
55 response = requests.get(url)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
56 if response.status_code == 200:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
57 tmp_filename = url.split("/")[-1]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
58 tmpfile = open(tmp_filename, "wb")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
59 tmpfile.write(response.content)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
60 tmpfile.close()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
61 shutil.copy(tmp_filename, "/tmp")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
62 tf = tarfile.open(tmp_filename)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
63 pl_path = tf.next().name
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
64 tf.extractall(output_directory)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
65 os.unlink(tmp_filename)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
66 os.rename(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
67 output_directory + "/" + pl_path + "/" + "pangoLEARN",
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
68 output_directory + "/" + tmp_filename,
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
69 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
70 shutil.rmtree(output_directory + "/" + pl_path)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
71 return tmp_filename
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
72 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
73 response.raise_for_status()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
74
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
75
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
76 def parse_date(d):
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
77 # Tries to parse the first 10 chars of d as a date, which currently
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
78 # succeeds for all pangolearn model releases.
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
79 return datetime.datetime.strptime(d[:10], "%Y-%m-%d")
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
80
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
81
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
82 if __name__ == "__main__":
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
83
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
84 parser = argparse.ArgumentParser()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
85 parser.add_argument("--testmode", default=False, action="store_true")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
86 parser.add_argument("--latest", default=False, action="store_true")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
87 parser.add_argument("--start_date", type=parse_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
88 parser.add_argument("--end_date", type=parse_date)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
89 parser.add_argument("--overwrite", default=False, action="store_true")
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
90 parser.add_argument('--pangolearn_format_version')
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
91 parser.add_argument("datatable_name")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
92 parser.add_argument("galaxy_datamanager_filename")
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
93 args = parser.parse_args()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
94
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
95 if args.testmode:
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
96 releases = filter_by_date([], start_date=args.start_date, end_date=args.end_date)
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
97 for release in releases:
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
98 print(release["tag_name"], release["tarball_url"].split("/")[-1], release["date"])
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
99 sys.exit(0)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
100
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
101 with open(args.galaxy_datamanager_filename) as fh:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
102 config = json.load(fh)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
103
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
104 output_directory = config.get("output_data", [{}])[0].get("extra_files_path", None)
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
105 data_manager_dict = {}
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
106 data_manager_dict["data_tables"] = config.get("data_tables", {})
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
107 data_manager_dict["data_tables"][args.datatable_name] = data_manager_dict[
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
108 "data_tables"
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
109 ].get(args.datatable_name, [])
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
110
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
111 # NOTE: the data_manager_dict["data_tables"][args.datatable_name] is not actually populated with the
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
112 # contents of the existing data table, so the "no-overwrite" logic and the
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
113 # only-download-what-we-don't-have logic does not in fact work. It is left but unused for now.
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
114 if not args.overwrite:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
115 existing_release_tags = set(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
116 [
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
117 el["value"]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
118 for el in data_manager_dict["data_tables"][args.datatable_name]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
119 ]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
120 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
121 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
122 existing_release_tags = set()
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
123 if args.latest:
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
124 releases = [next(get_model_list(existing_release_tags))]
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
125 else:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
126 releases = filter_by_date(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
127 existing_release_tags, start_date=args.start_date, end_date=args.end_date
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
128 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
129 releases_to_download = [
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
130 release
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
131 for release in releases
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
132 if release["tag_name"] not in existing_release_tags
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
133 ]
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
134 for release in releases_to_download:
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
135 fname = download_and_unpack(release["tarball_url"], output_directory)
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
136 if args.pangolearn_format_version is not None:
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
137 version = args.pangolearn_format_version
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
138 else:
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
139 # 2021-05-27 was the first release of pangoLEARN for pangolin 3, which changed DB format
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
140 if release["date"] >= datetime.datetime(2021, 5, 27):
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
141 version = '3.0'
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
142 else:
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
143 version = '1.0'
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
144 data_manager_dict["data_tables"][args.datatable_name].append(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
145 dict(
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
146 value=release["tag_name"],
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
147 description=release["name"],
1
f7f2e94d68fc "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 1f7b91c7bbd72d30ab47c64d0796ce9825edcaf9"
iuc
parents: 0
diff changeset
148 format_version=version,
4
4a37568620a1 "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit fd2566abd51c88847437d38a5abea8703b8ee034"
iuc
parents: 3
diff changeset
149 path=output_directory + "/" + fname,
0
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
150 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
151 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
152 data_manager_dict["data_tables"][args.datatable_name].sort(
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
153 key=operator.itemgetter("value"), reverse=True
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
154 )
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
155 with open(args.galaxy_datamanager_filename, "w") as fh:
e5c31cab174c "planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/data_managers/data_manager_pangolearn commit 68adfad76cdb5ff13ec9fb49895a3cde2c502514"
iuc
parents:
diff changeset
156 json.dump(data_manager_dict, fh, indent=2, sort_keys=True)