Mercurial > repos > fubar > tool_factory_2

comparison toolfactory/ToolFactory_tester.xml @ 3:1c652687a08f draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Uploaded
author fubar
date Fri, 30 Apr 2021 07:06:57 +0000
parents
children
comparison
equal deleted inserted replaced
2:5fc0c9a93072 3:1c652687a08f
1 <tool name="toolfactory_tester" id="toolfactory_tester" version="1">
2 <!--Source in git at: https://github.com/fubar2/toolfactory-->
3 <!--Created by admin@galaxy.org at 23/04/2021 10:25:58 using the Galaxy Tool Factory.-->
4 <description>Test an untested tool and update it</description>
5 <requirements>
6 <requirement type="package" version="2.30.2">git</requirement>
7 <requirement type="package" version="0.74.3">planemo</requirement> </requirements>
8 <stdio>
9 <exit_code range="1:" level="fatal"/>
10 </stdio>
11 <version_command><![CDATA[echo "1"]]></version_command>
12 <command><![CDATA[
13 mkdir 'TF_run_report';
14 python
15 $runme
16 --in_tool_archive
17 $in_tool_archive
18 --new_tested_tool_archive
19 $new_tested_tool_archive
20 --galaxy_root
21 "$__root_dir__"
22 >
23 "TF_run_report/${in_tool_archive.name}_test_log.txt"
24 ]]></command>
25 <configfiles>
26 <configfile name="runme"><![CDATA[#raw
27
28 # see https://github.com/fubar2/toolfactory
29 #
30 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
31 #
32 # all rights reserved
33 # Licensed under the LGPL
34 # suggestions for improvement and bug fixes welcome at
35 # https://github.com/fubar2/toolfactory
36 #
37 # July 2020: BCC was fun and I feel like rip van winkle after 5 years.
38 # Decided to
39 # 1. Fix the toolfactory so it works - done for simplest case
40 # 2. Fix planemo so the toolfactory function works
41 # 3. Rewrite bits using galaxyxml functions where that makes sense - done
42
43 import argparse
44 import copy
45 import os
46 import subprocess
47 import shutil
48 import sys
49 import tarfile
50 import tempfile
51 import time
52 import xml.etree.ElementTree as ET
53
54
55 myversion = "V2.2 April 2021"
56 verbose = True
57 debug = True
58 toolFactoryURL = "https://github.com/fubar2/toolfactory"
59
60 def timenow():
61 """return current time as a string"""
62 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
63
64 class ToolTester():
65 # uncompress passed tar, run planemo and rebuild a new tarball with tests
66
67 def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None):
68 self.args = args
69 self.new_tool_archive = new_tool_archive
70 assert tarfile.is_tarfile(in_tool_archive)
71 # this is not going to go well with arbitrary names.
72 tff = tarfile.open(in_tool_archive, "r:*")
73 flist = tff.getnames()
74 ourdir = os.path.commonpath(flist) # eg pyrevpos
75 self.tool_name = ourdir
76 ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir]
77 # planemo_test/planemo_test.xml
78 assert len(ourxmls) > 0
79 self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls]
80 res = tff.extractall()
81 self.update_tests(ourdir)
82 tff.close()
83 self.tooloutdir = "./tfout"
84 self.repdir = "./TF_run_report"
85 self.testdir = os.path.join(self.tooloutdir, "test-data")
86 if not os.path.exists(self.tooloutdir):
87 os.mkdir(self.tooloutdir)
88 if not os.path.exists(self.testdir):
89 os.mkdir(self.testdir)
90 if not os.path.exists(self.repdir):
91 os.mkdir(self.repdir)
92 self.makeTool()
93 self.moveRunOutputs()
94 self.makeToolTar()
95
96 def call_planemo(self,xmlpath,ourdir):
97 penv = os.environ
98 penv['HOME'] = os.path.join(self.args.galaxy_root,'planemo')
99 newpath = f"{penv['HOME']}:{penv['PATH']}"
100 penv['PATH'] = newpath
101 penv["GALAXY_VIRTUAL_ENV"] = os.path.join(self.args.galaxy_root,'.venv')
102 penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache')
103 toolfile = os.path.split(xmlpath)[1]
104 tool_name = self.tool_name
105 tool_test_output = f"{tool_name}_planemo_test_report.html"
106 cll = [
107 "planemo",
108 "test",
109 "--no_cleanup",
110 "--test_data",
111 os.path.abspath(ourdir),
112 "--test_output",
113 os.path.abspath(self.tool_test_output),
114 "--galaxy_root",
115 self.args.galaxy_root,
116 "--update_test_data",
117 os.path.abspath(xmlpath),
118 ]
119 print(cll)
120 p = subprocess.run(
121 cll,
122 #capture_output=True,
123 encoding='utf8',
124 cwd = os.path.abspath(self.tool_name),
125 env = penv,
126 shell=False,
127 )
128 return p
129
130 def makeTool(self):
131 """write xmls and input samples into place"""
132 for xreal in self.ourxmls:
133 x = os.path.split(xreal)[1]
134 xout = os.path.join(self.tooloutdir,x)
135 shutil.copyfile(xreal, xout)
136
137 def makeToolTar(self):
138 """move outputs into test-data and prepare the tarball"""
139 excludeme = "_planemo_test_report.html"
140
141 def exclude_function(tarinfo):
142 filename = tarinfo.name
143 return None if filename.endswith(excludeme) else tarinfo
144
145 newtar = 'new_%s_toolshed.gz' % self.tool_name
146 ttf = tarfile.open(newtar, "w:gz")
147 ttf.add(name=self.tool_name,
148 arcname=self.tool_name,
149 filter=exclude_function)
150 ttf.close()
151 shutil.copyfile(newtar, self.new_tool_archive)
152
153 def moveRunOutputs(self):
154 """need to move planemo or run outputs into toolfactory collection"""
155 with os.scandir(self.tooloutdir) as outs:
156 for entry in outs:
157 if not entry.is_file():
158 continue
159 if "." in entry.name:
160 _, ext = os.path.splitext(entry.name)
161 if ext in [".tgz", ".json"]:
162 continue
163 if ext in [".yml", ".xml", ".yaml"]:
164 newname = f"{entry.name.replace('.','_')}.txt"
165 else:
166 newname = entry.name
167 else:
168 newname = f"{entry.name}.txt"
169 dest = os.path.join(self.repdir, newname)
170 src = os.path.join(self.tooloutdir, entry.name)
171 shutil.copyfile(src, dest)
172 with os.scandir('.') as outs:
173 for entry in outs:
174 if not entry.is_file() or entry.name == "conda_activate.log":
175 continue
176 if "." in entry.name:
177 _, ext = os.path.splitext(entry.name)
178 if ext in [".yml", ".xml", ".yaml"]:
179 newname = f"{entry.name.replace('.','_')}.txt"
180 else:
181 newname = entry.name
182 else:
183 newname = f"{entry.name}.txt"
184 dest = os.path.join(self.repdir, newname)
185 src =entry.name
186 shutil.copyfile(src, dest)
187 if True or self.args.include_tests:
188 with os.scandir(self.testdir) as outs:
189 for entry in outs:
190 if (not entry.is_file()) or entry.name.endswith(
191 "_planemo_test_report.html"
192 ):
193 continue
194 if "." in entry.name:
195 _, ext = os.path.splitext(entry.name)
196 if ext in [".tgz", ".json"]:
197 continue
198 if ext in [".yml", ".xml", ".yaml"]:
199 newname = f"{entry.name.replace('.','_')}.txt"
200 else:
201 newname = entry.name
202 else:
203 newname = f"{entry.name}.txt"
204 dest = os.path.join(self.repdir, newname)
205 src = os.path.join(self.testdir, entry.name)
206 shutil.copyfile(src, dest)
207
208
209 def update_tests(self,ourdir):
210 for xmlf in self.ourxmls:
211 capture = self.call_planemo(xmlf,ourdir)
212
213 def main():
214 """
215 This is a Galaxy wrapper.
216 It expects to be called by a special purpose tool.xml
217
218 """
219 parser = argparse.ArgumentParser()
220 a = parser.add_argument
221 a("--in_tool_archive", default=None)
222 a("--new_tested_tool_archive", default=None)
223 a("--galaxy_root", default="/home/ross/gal21/")
224 args = parser.parse_args()
225 print('Hello from',os.getcwd())
226 tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive)
227
228 if __name__ == "__main__":
229 main()
230
231
232 #end raw]]></configfile>
233 </configfiles>
234 <inputs>
235 <param name="new_tool_name" value="" type="hidden"/>
236 <param name="in_tool_archive" type="data" optional="false" label="Select a no_test tarfile to test and update for a toolshed" help="" format="toolshed.gz" multiple="false"/>
237 </inputs>
238 <outputs>
239 <data name="new_tested_tool_archive" format="toolshed.gz" label="${in_tool_archive.name.split('_')[0]}_tested_toolshed.gz" hidden="false"/>
240 <collection name="TF_run_report" type="list" label="${in_tool_archive.name} test Run reports">
241 <discover_datasets pattern="__name_and_ext__" directory="TF_run_report" visible="false"/>
242 </collection>
243 </outputs>
244 <tests>
245 <test>
246 <output name="new_tested_tool_archive" value="new_tested_tool_archive_sample" compare="sim_size" delta_frac="0.5"/>
247 <param name="in_tool_archive" value="in_tool_archive_sample"/>
248 <output_collection name="TF_run_report"/>
249 </test>
250 </tests>
251 <help><![CDATA[
252
253 **What it Does**
254
255 ------
256
257 Script::
258
259 # see https://github.com/fubar2/toolfactory
260 #
261 # copyright ross lazarus (ross stop lazarus at gmail stop com) May 2012
262 #
263 # all rights reserved
264 # Licensed under the LGPL
265 # suggestions for improvement and bug fixes welcome at
266 # https://github.com/fubar2/toolfactory
267
268 import argparse
269 import copy
270 import os
271 import subprocess
272 import shutil
273 import sys
274 import tarfile
275 import tempfile
276 import time
277 import xml.etree.ElementTree as ET
278
279
280 myversion = "V2.2 April 2021"
281 verbose = True
282 debug = True
283 toolFactoryURL = "https://github.com/fubar2/toolfactory"
284
285 def timenow():
286 """return current time as a string"""
287 return time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(time.time()))
288
289 class ToolTester():
290 # uncompress passed tar, run planemo and rebuild a new tarball with tests
291
292 def __init__(self, args=None, in_tool_archive='/galaxy-central/tools/newtool/newtool_toolshed.gz', new_tool_archive=None):
293 self.args = args
294 self.new_tool_archive = new_tool_archive
295 assert tarfile.is_tarfile(in_tool_archive)
296 # this is not going to go well with arbitrary names.
297 tff = tarfile.open(in_tool_archive, "r:*")
298 flist = tff.getnames()
299 ourdir = os.path.commonpath(flist) # eg pyrevpos
300 self.tool_name = ourdir
301 ourxmls = [x for x in flist if x.lower().endswith('.xml') and os.path.split(x)[0] == ourdir]
302 assert len(ourxmls) > 0
303 self.ourxmls = ourxmls # [os.path.join(tool_path,x) for x in ourxmls]
304 res = tff.extractall()
305 self.update_tests(ourdir)
306 tff.close()
307 self.tooloutdir = "./tfout"
308 self.repdir = "./TF_run_report"
309 self.testdir = os.path.join(self.tooloutdir, "test-data")
310 if not os.path.exists(self.tooloutdir):
311 os.mkdir(self.tooloutdir)
312 if not os.path.exists(self.testdir):
313 os.mkdir(self.testdir)
314 if not os.path.exists(self.repdir):
315 os.mkdir(self.repdir)
316 self.makeTool()
317 self.moveRunOutputs()
318 self.makeToolTar()
319
320 def call_planemo(self,xmlpath,ourdir):
321 penv = os.environ
322 penv["PIP_CACHE_DIR"] = os.path.join(self.args.galaxy_root,'pipcache')
323 toolfile = os.path.split(xmlpath)[1]
324 tool_name = self.tool_name
325 tool_test_output = f"{tool_name}_planemo_test_report.html"
326 cll = [
327 "planemo",
328 "test",
329 "--biocontainers",
330 "--test_output",
331 os.path.abspath(tool_test_output),
332 "--galaxy_root",
333 self.args.galaxy_root,
334 "--update_test_data",
335 os.path.abspath(xmlpath),
336 ]
337 print(cll)
338 p = subprocess.run(
339 cll,
340 #capture_output=True,
341 encoding='utf8',
342 env = penv,
343 shell=False,
344 )
345 return p
346
347 def makeTool(self):
348 """write xmls and input samples into place"""
349 for xreal in self.ourxmls:
350 x = os.path.split(xreal)[1]
351 xout = os.path.join(self.tooloutdir,x)
352 shutil.copyfile(xreal, xout)
353
354 def makeToolTar(self):
355 """move outputs into test-data and prepare the tarball"""
356 excludeme = "_planemo_test_report.html"
357
358 def exclude_function(tarinfo):
359 filename = tarinfo.name
360 return None if filename.endswith(excludeme) else tarinfo
361
362 newtar = 'new_%s_toolshed.gz' % self.tool_name
363 ttf = tarfile.open(newtar, "w:gz")
364 ttf.add(name=self.tool_name,
365 arcname=self.tool_name,
366 filter=exclude_function)
367 ttf.close()
368 shutil.copyfile(newtar, self.new_tool_archive)
369
370 def moveRunOutputs(self):
371 """need to move planemo or run outputs into toolfactory collection"""
372 with os.scandir(self.tooloutdir) as outs:
373 for entry in outs:
374 if not entry.is_file():
375 continue
376 if "." in entry.name:
377 _, ext = os.path.splitext(entry.name)
378 if ext in [".tgz", ".json"]:
379 continue
380 if ext in [".yml", ".xml", ".yaml"]:
381 newname = f"{entry.name.replace('.','_')}.txt"
382 else:
383 newname = entry.name
384 else:
385 newname = f"{entry.name}.txt"
386 dest = os.path.join(self.repdir, newname)
387 src = os.path.join(self.tooloutdir, entry.name)
388 shutil.copyfile(src, dest)
389 with os.scandir('.') as outs:
390 for entry in outs:
391 if not entry.is_file():
392 continue
393 if "." in entry.name:
394 _, ext = os.path.splitext(entry.name)
395 if ext in [".yml", ".xml", ".yaml"]:
396 newname = f"{entry.name.replace('.','_')}.txt"
397 else:
398 newname = entry.name
399 else:
400 newname = f"{entry.name}.txt"
401 dest = os.path.join(self.repdir, newname)
402 src =entry.name
403 shutil.copyfile(src, dest)
404 if True or self.args.include_tests:
405 with os.scandir(self.testdir) as outs:
406 for entry in outs:
407 if (not entry.is_file()) or entry.name.endswith(
408 "_planemo_test_report.html"
409 ):
410 continue
411 if "." in entry.name:
412 _, ext = os.path.splitext(entry.name)
413 if ext in [".tgz", ".json"]:
414 continue
415 if ext in [".yml", ".xml", ".yaml"]:
416 newname = f"{entry.name.replace('.','_')}.txt"
417 else:
418 newname = entry.name
419 else:
420 newname = f"{entry.name}.txt"
421 dest = os.path.join(self.repdir, newname)
422 src = os.path.join(self.testdir, entry.name)
423 shutil.copyfile(src, dest)
424
425
426 def update_tests(self,ourdir):
427 for xmlf in self.ourxmls:
428 capture = self.call_planemo(xmlf,ourdir)
429
430 def main():
431 """
432 This is a Galaxy wrapper.
433 It expects to be called by a special purpose tool.xml
434
435 """
436 parser = argparse.ArgumentParser()
437 a = parser.add_argument
438 a("--in_tool_archive", default=None)
439 a("--new_tested_tool_archive", default=None)
440 a("--galaxy_root", default="/home/ross/gal21/")
441 args = parser.parse_args()
442 print('Hello from',os.getcwd())
443 tt = ToolTester(args=args, in_tool_archive=args.in_tool_archive, new_tool_archive=args.new_tested_tool_archive)
444
445 if __name__ == "__main__":
446 main()
447
448
449 ]]></help>
450 <citations>
451 <citation type="doi">10.1093/bioinformatics/bts573</citation>
452 </citations>
453 </tool>
454