Mercurial > repos > tduigou > data_manager_cache

annotate data_manager/cache_fetcher.py @ 0:e0b92d203870 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
author tduigou
date Mon, 04 Jul 2022 13:28:30 +0000
parents
children 35c33747b9e3
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
1 from os import (
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
2 path as os_path,
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
3 mkdir as os_mkdir
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
4 )
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
5
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
6 from requests import get as r_get
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
7
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
8 from tempfile import (
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
9 NamedTemporaryFile
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
10 )
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
11
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
12 import argparse
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
13
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
14 def download(
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
15 url: str,
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
16 file: str = ""
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
17 ) -> str:
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
18 """
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
19 Download a file from 'url' and save it as 'file'.
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
20 Parameters:
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
21 url -- URL the file is downloaded from
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
22 file -- (Optional) filename the downloaded file is saved into (default: "")
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
23 Returns:
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
24 A filename where the downloaded file has stored into
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
25 """
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
26 r = r_get(url)
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
27 if not file:
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
28 f = NamedTemporaryFile(
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
29 mode='wb',
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
30 delete=False
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
31 )
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
32 file = f.name
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
33 else:
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
34 f = open(file, 'wb')
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
35 f.write(r.content)
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
36 f.close()
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
37 return file
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
38
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
39 parser = argparse.ArgumentParser(description="Download a cache file")
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
40 parser.add_argument('-u','--url', required=True, default=None, type=str, help="URL the file is downloaded from")
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
41 parser.add_argument('-o','--outfile', required=True, default=None, type=str, help="A filename where the downloaded file has stored into")
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
42
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
43 args = parser.parse_args()
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
44
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
45 url= args.url #"https://gitlab.com/breakthewall/rrCache-data/-/raw/master/"
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
46 filename= os_path.basename(args.outfile) #"cid_strc.json.gz"
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
47 cache_dir=os_path.dirname(args.outfile) #'${GALAXY_DATA_MANAGER_DATA_PATH}'+'/rpextractsink/cache/'
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
48 full_filename=os_path.join(cache_dir,filename) #'${GALAXY_DATA_MANAGER_DATA_PATH}'+'/rpextractsink/cache/cid_strc.json.gz'
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
49
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
50 if not os_path.isdir(cache_dir):
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
51 os_mkdir(cache_dir)
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
52
e0b92d203870 "planemo upload commit f40274f6b9f6a15eb4022aab21286d4c96cd8475-dirty"
tduigou
parents:
diff changeset
53 download(url+filename, full_filename)