Mercurial > repos > mvdbeek > docker_toolfactory_alpha
annotate DockerToolFactory.py @ 4:de4889098f45 default tip
Allow whitespace in gene name and fix missing creation of output section.
| author | mvdbeek |
|---|---|
| date | Wed, 14 Jan 2015 17:19:16 +0100 |
| parents | 5b930e77b1f3 |
| children |
| rev | line source |
|---|---|
| 0 | 1 # DockerToolFactory.py |
| 2 # see https://bitbucket.org/mvdbeek/DockerToolFactory | |
| 3 | |
| 4 import sys | |
| 5 import shutil | |
| 6 import subprocess | |
| 7 import os | |
| 8 import time | |
| 9 import tempfile | |
| 10 import argparse | |
| 11 import tarfile | |
| 12 import re | |
| 13 import shutil | |
| 14 import math | |
| 15 import fileinput | |
| 16 from os.path import abspath | |
| 17 | |
| 18 progname = os.path.split(sys.argv[0])[1] | |
| 19 myversion = 'V001.1 March 2014' | |
| 20 verbose = False | |
| 21 debug = False | |
| 22 toolFactoryURL = 'https://bitbucket.org/fubar/galaxytoolfactory' | |
| 23 | |
| 24 # if we do html we need these dependencies specified in a tool_dependencies.xml file and referred to in the generated | |
| 25 # tool xml | |
| 26 toolhtmldepskel = """<?xml version="1.0"?> | |
| 27 <tool_dependency> | |
| 28 <package name="ghostscript" version="9.10"> | |
| 29 <repository name="package_ghostscript_9_10" owner="devteam" prior_installation_required="True" /> | |
| 30 </package> | |
| 31 <package name="graphicsmagick" version="1.3.18"> | |
| 32 <repository name="package_graphicsmagick_1_3" owner="iuc" prior_installation_required="True" /> | |
| 33 </package> | |
| 34 <readme> | |
| 35 %s | |
| 36 </readme> | |
| 37 </tool_dependency> | |
| 38 """ | |
| 39 | |
| 40 protorequirements = """<requirements> | |
| 41 <requirement type="package" version="9.10">ghostscript</requirement> | |
| 42 <requirement type="package" version="1.3.18">graphicsmagick</requirement> | |
| 43 <container type="docker">toolfactory/custombuild:%s</container> | |
| 44 </requirements>""" | |
| 45 | |
| 46 def timenow(): | |
| 47 """return current time as a string | |
| 48 """ | |
| 49 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time())) | |
| 50 | |
| 51 html_escape_table = { | |
| 52 "&": "&", | |
| 53 ">": ">", | |
| 54 "<": "<", | |
| 55 "$": "\$" | |
| 56 } | |
| 57 | |
| 58 def html_escape(text): | |
| 59 """Produce entities within text.""" | |
| 60 return "".join(html_escape_table.get(c,c) for c in text) | |
| 61 | |
| 62 def cmd_exists(cmd): | |
| 63 return subprocess.call("type " + cmd, shell=True, | |
| 64 stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 | |
| 65 | |
| 66 def edit_dockerfile(dockerfile): | |
| 67 '''we have to change the userid of galaxy inside the container to the id with which the tool is run, | |
| 68 otherwise we have a mismatch in the file permissions inside the container''' | |
| 69 uid=os.getuid() | |
| 70 for line in fileinput.FileInput(dockerfile, inplace=1): | |
|
2
5b930e77b1f3
Better readability of Dockerfile, fix editing of userid for Dockerfile in DockerToolFactory.py.
mvdbeek
parents:
1
diff
changeset
|
71 sys.stdout.write(re.sub("RUN adduser galaxy.*", "RUN adduser galaxy -u {0}\n".format(uid), line)) |
| 0 | 72 |
| 73 def build_docker(dockerfile, docker_client, image_tag='base'): | |
| 74 '''Given the path to a dockerfile, and a docker_client, build the image, if it does not | |
| 75 exist yet.''' | |
| 76 image_id='toolfactory/custombuild:'+image_tag | |
| 77 existing_images=", ".join(["".join(d['RepoTags']) for d in docker_client.images()]) | |
| 78 if image_id in existing_images: | |
| 79 print 'docker container exists, skipping build' | |
| 80 return image_id | |
| 81 print "Building Docker image, using Dockerfile:{0}".format(dockerfile) | |
| 1 | 82 build_process=docker_client.build(fileobj=open(dockerfile, 'r'), tag=image_id) |
| 0 | 83 print "succesfully dispatched docker build process, building now" |
| 84 build_log=[line for line in build_process] #will block until image is built. | |
| 85 return image_id | |
| 86 | |
| 87 def construct_bind(host_path, container_path=False, binds=None, ro=True): | |
| 88 #TODO remove container_path if it's alwyas going to be the same as host_path | |
| 89 '''build or extend binds dictionary with container path. binds is used | |
| 90 to mount all files using the docker-py client.''' | |
| 91 if not binds: | |
| 92 binds={} | |
| 93 if isinstance(host_path, list): | |
| 94 for k,v in enumerate(host_path): | |
| 95 if not container_path: | |
| 96 container_path=host_path[k] | |
| 97 binds[host_path[k]]={'bind':container_path, 'ro':ro} | |
| 98 container_path=False #could be more elegant | |
| 99 return binds | |
| 100 else: | |
| 101 if not container_path: | |
| 102 container_path=host_path | |
| 103 binds[host_path]={'bind':container_path, 'ro':ro} | |
| 104 return binds | |
| 105 | |
| 106 def switch_to_docker(opts): | |
| 107 import docker #need local import, as container does not have docker-py | |
| 108 docker_client=docker.Client() | |
| 109 toolfactory_path=abspath(sys.argv[0]) | |
| 110 dockerfile=os.path.dirname(toolfactory_path)+'/Dockerfile' | |
| 111 edit_dockerfile(dockerfile) | |
| 112 image_id=build_docker(dockerfile, docker_client) | |
| 113 binds=construct_bind(host_path=opts.script_path, ro=False) | |
| 114 binds=construct_bind(binds=binds, host_path=abspath(opts.output_dir), ro=False) | |
| 115 if len(opts.input_tab)>0: | |
| 116 binds=construct_bind(binds=binds, host_path=opts.input_tab, ro=True) | |
| 117 if not opts.output_tab == 'None': | |
| 118 binds=construct_bind(binds=binds, host_path=opts.output_tab, ro=False) | |
| 119 if opts.make_HTML: | |
| 120 binds=construct_bind(binds=binds, host_path=opts.output_html, ro=False) | |
| 121 if opts.make_Tool: | |
| 122 binds=construct_bind(binds=binds, host_path=opts.new_tool, ro=False) | |
| 123 binds=construct_bind(binds=binds, host_path=opts.help_text, ro=True) | |
| 124 binds=construct_bind(binds=binds, host_path=toolfactory_path) | |
| 125 volumes=binds.keys() | |
| 126 sys.argv=[abspath(opts.output_dir) if sys.argv[i-1]=='--output_dir' else arg for i,arg in enumerate(sys.argv)] ##inject absolute path of working_dir | |
| 127 cmd=['python', '-u']+sys.argv+['--dockerized', '1'] | |
| 128 container=docker_client.create_container( | |
| 129 image=image_id, | |
| 130 user='galaxy', | |
| 131 volumes=volumes, | |
| 132 command=cmd | |
| 133 ) | |
| 134 docker_client.start(container=container[u'Id'], binds=binds) | |
| 135 docker_client.wait(container=container[u'Id']) | |
| 136 logs=docker_client.logs(container=container[u'Id']) | |
| 137 print "".join([log for log in logs]) | |
| 138 | |
| 139 class ScriptRunner: | |
| 140 """class is a wrapper for an arbitrary script | |
| 141 """ | |
| 142 | |
| 143 def __init__(self,opts=None,treatbashSpecial=True, image_tag='base'): | |
| 144 """ | |
| 145 cleanup inputs, setup some outputs | |
| 146 | |
| 147 """ | |
| 148 self.opts = opts | |
| 149 self.useGM = cmd_exists('gm') | |
| 150 self.useIM = cmd_exists('convert') | |
| 151 self.useGS = cmd_exists('gs') | |
| 152 self.temp_warned = False # we want only one warning if $TMP not set | |
| 153 self.treatbashSpecial = treatbashSpecial | |
| 154 self.image_tag = image_tag | |
| 155 os.chdir(abspath(opts.output_dir)) | |
| 156 self.thumbformat = 'png' | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
157 self.toolname_sanitized = re.sub('[^a-zA-Z0-9_]+', '_', opts.tool_name) # a sanitizer now does this but.. |
|
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
158 self.toolname = opts.tool_name |
| 0 | 159 self.toolid = self.toolname |
| 160 self.myname = sys.argv[0] # get our name because we write ourselves out as a tool later | |
| 161 self.pyfile = self.myname # crude but efficient - the cruft won't hurt much | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
162 self.xmlfile = '%s.xml' % self.toolname_sanitized |
| 0 | 163 s = open(self.opts.script_path,'r').readlines() |
| 164 s = [x.rstrip() for x in s] # remove pesky dos line endings if needed | |
| 165 self.script = '\n'.join(s) | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
166 fhandle,self.sfile = tempfile.mkstemp(prefix=self.toolname_sanitized,suffix=".%s" % (opts.interpreter)) |
| 0 | 167 tscript = open(self.sfile,'w') # use self.sfile as script source for Popen |
| 168 tscript.write(self.script) | |
| 169 tscript.close() | |
| 170 self.indentedScript = '\n'.join([' %s' % html_escape(x) for x in s]) # for restructured text in help | |
| 171 self.escapedScript = '\n'.join([html_escape(x) for x in s]) | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
172 self.elog = os.path.join(self.opts.output_dir,"%s_error.log" % self.toolname_sanitized) |
| 0 | 173 if opts.output_dir: # may not want these complexities |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
174 self.tlog = os.path.join(self.opts.output_dir,"%s_runner.log" % self.toolname_sanitized) |
|
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
175 art = '%s.%s' % (self.toolname_sanitized,opts.interpreter) |
| 0 | 176 artpath = os.path.join(self.opts.output_dir,art) # need full path |
| 177 artifact = open(artpath,'w') # use self.sfile as script source for Popen | |
| 178 artifact.write(self.script) | |
| 179 artifact.close() | |
| 180 self.cl = [] | |
| 181 self.html = [] | |
| 182 a = self.cl.append | |
| 183 a(opts.interpreter) | |
| 184 if self.treatbashSpecial and opts.interpreter in ['bash','sh']: | |
| 185 a(self.sfile) | |
| 186 else: | |
| 187 a('-') # stdin | |
| 188 for input in opts.input_tab: | |
| 189 a(input) | |
| 190 if opts.output_tab == 'None': #If tool generates only HTML, set output name to toolname | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
191 a(str(self.toolname_sanitized)+'.out') |
| 0 | 192 a(opts.output_tab) |
| 193 for param in opts.additional_parameters: | |
| 194 param, value=param.split(',') | |
| 195 a('--'+param) | |
| 196 a(value) | |
| 197 #print self.cl | |
| 198 self.outFormats = opts.output_format | |
| 199 self.inputFormats = [formats for formats in opts.input_formats] | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
200 self.test1Input = '%s_test1_input.xls' % self.toolname_sanitized |
|
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
201 self.test1Output = '%s_test1_output.xls' % self.toolname_sanitized |
|
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
202 self.test1HTML = '%s_test1_output.html' % self.toolname_sanitized |
| 0 | 203 |
| 204 def makeXML(self): | |
| 205 """ | |
| 206 Create a Galaxy xml tool wrapper for the new script as a string to write out | |
| 207 fixme - use templating or something less fugly than this example of what we produce | |
| 208 | |
| 209 <tool id="reverse" name="reverse" version="0.01"> | |
| 210 <description>a tabular file</description> | |
| 211 <command interpreter="python"> | |
| 212 reverse.py --script_path "$runMe" --interpreter "python" | |
| 213 --tool_name "reverse" --input_tab "$input1" --output_tab "$tab_file" | |
| 214 </command> | |
| 215 <inputs> | |
| 216 <param name="input1" type="data" format="tabular" label="Select a suitable input file from your history"/> | |
| 217 | |
| 218 </inputs> | |
| 219 <outputs> | |
| 220 <data format=opts.output_format name="tab_file"/> | |
| 221 | |
| 222 </outputs> | |
| 223 <help> | |
| 224 | |
| 225 **What it Does** | |
| 226 | |
| 227 Reverse the columns in a tabular file | |
| 228 | |
| 229 </help> | |
| 230 <configfiles> | |
| 231 <configfile name="runMe"> | |
| 232 | |
| 233 # reverse order of columns in a tabular file | |
| 234 import sys | |
| 235 inp = sys.argv[1] | |
| 236 outp = sys.argv[2] | |
| 237 i = open(inp,'r') | |
| 238 o = open(outp,'w') | |
| 239 for row in i: | |
| 240 rs = row.rstrip().split('\t') | |
| 241 rs.reverse() | |
| 242 o.write('\t'.join(rs)) | |
| 243 o.write('\n') | |
| 244 i.close() | |
| 245 o.close() | |
| 246 | |
| 247 | |
| 248 </configfile> | |
| 249 </configfiles> | |
| 250 </tool> | |
| 251 | |
| 252 """ | |
| 253 newXML="""<tool id="%(toolid)s" name="%(toolname)s" version="%(tool_version)s"> | |
| 254 %(tooldesc)s | |
| 255 %(requirements)s | |
| 256 <command interpreter="python"> | |
| 257 %(command)s | |
| 258 </command> | |
| 259 <inputs> | |
| 260 %(inputs)s | |
| 261 </inputs> | |
| 262 <outputs> | |
| 263 %(outputs)s | |
| 264 </outputs> | |
| 265 <configfiles> | |
| 266 <configfile name="runMe"> | |
| 267 %(script)s | |
| 268 </configfile> | |
| 269 </configfiles> | |
| 270 | |
| 271 %(tooltests)s | |
| 272 | |
| 273 <help> | |
| 274 | |
| 275 %(help)s | |
| 276 | |
| 277 </help> | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
278 </tool>""" # needs a dict with toolname, toolname_sanitized, toolid, interpreter, scriptname, command, inputs as a multi line string ready to write, outputs ditto, help ditto |
| 0 | 279 |
| 280 newCommand=""" | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
281 %(toolname_sanitized)s.py --script_path "$runMe" --interpreter "%(interpreter)s" |
| 0 | 282 --tool_name "%(toolname)s" %(command_inputs)s %(command_outputs)s """ |
| 283 # may NOT be an input or htmlout - appended later | |
| 284 tooltestsTabOnly = """ | |
| 285 <tests> | |
| 286 <test> | |
| 287 <param name="input1" value="%(test1Input)s" ftype="tabular"/> | |
| 288 <param name="runMe" value="$runMe"/> | |
| 289 <output name="tab_file" file="%(test1Output)s" ftype="tabular"/> | |
| 290 </test> | |
| 291 </tests> | |
| 292 """ | |
| 293 tooltestsHTMLOnly = """ | |
| 294 <tests> | |
| 295 <test> | |
| 296 <param name="input1" value="%(test1Input)s" ftype="tabular"/> | |
| 297 <param name="runMe" value="$runMe"/> | |
| 298 <output name="html_file" file="%(test1HTML)s" ftype="html" lines_diff="5"/> | |
| 299 </test> | |
| 300 </tests> | |
| 301 """ | |
| 302 tooltestsBoth = """<tests> | |
| 303 <test> | |
| 304 <param name="input1" value="%(test1Input)s" ftype="tabular"/> | |
| 305 <param name="runMe" value="$runMe"/> | |
| 306 <output name="tab_file" file="%(test1Output)s" ftype="tabular" /> | |
| 307 <output name="html_file" file="%(test1HTML)s" ftype="html" lines_diff="10"/> | |
| 308 </test> | |
| 309 </tests> | |
| 310 """ | |
| 311 xdict = {} | |
| 312 #xdict['requirements'] = '' | |
| 313 #if self.opts.make_HTML: | |
| 314 xdict['requirements'] = protorequirements % self.image_tag | |
| 315 xdict['tool_version'] = self.opts.tool_version | |
| 316 xdict['test1Input'] = self.test1Input | |
| 317 xdict['test1HTML'] = self.test1HTML | |
| 318 xdict['test1Output'] = self.test1Output | |
| 319 if self.opts.make_HTML and self.opts.output_tab <> 'None': | |
| 320 xdict['tooltests'] = tooltestsBoth % xdict | |
| 321 elif self.opts.make_HTML: | |
| 322 xdict['tooltests'] = tooltestsHTMLOnly % xdict | |
| 323 else: | |
| 324 xdict['tooltests'] = tooltestsTabOnly % xdict | |
| 325 xdict['script'] = self.escapedScript | |
| 326 # configfile is least painful way to embed script to avoid external dependencies | |
| 327 # but requires escaping of <, > and $ to avoid Mako parsing | |
| 328 if self.opts.help_text: | |
| 329 helptext = open(self.opts.help_text,'r').readlines() | |
| 330 helptext = [html_escape(x) for x in helptext] # must html escape here too - thanks to Marius van den Beek | |
| 331 xdict['help'] = ''.join([x for x in helptext]) | |
| 332 else: | |
| 333 xdict['help'] = 'Please ask the tool author (%s) for help as none was supplied at tool generation\n' % (self.opts.user_email) | |
| 334 coda = ['**Script**','Pressing execute will run the following code over your input file and generate some outputs in your history::'] | |
| 335 coda.append('\n') | |
| 336 coda.append(self.indentedScript) | |
| 337 coda.append('\n**Attribution**\nThis Galaxy tool was created by %s at %s\nusing the Galaxy Tool Factory.\n' % (self.opts.user_email,timenow())) | |
| 338 coda.append('See %s for details of that project' % (toolFactoryURL)) | |
| 339 coda.append('Please cite: Creating re-usable tools from scripts: The Galaxy Tool Factory. Ross Lazarus; Antony Kaspi; Mark Ziemann; The Galaxy Team. ') | |
| 340 coda.append('Bioinformatics 2012; doi: 10.1093/bioinformatics/bts573\n') | |
| 341 xdict['help'] = '%s\n%s' % (xdict['help'],'\n'.join(coda)) | |
| 342 if self.opts.tool_desc: | |
| 343 xdict['tooldesc'] = '<description>%s</description>' % self.opts.tool_desc | |
| 344 else: | |
| 345 xdict['tooldesc'] = '' | |
| 346 xdict['command_outputs'] = '' | |
| 347 xdict['outputs'] = '' | |
| 348 if self.opts.input_tab <> 'None': | |
| 349 xdict['command_inputs'] = '--input_tab' | |
| 350 xdict['inputs']='' | |
| 351 for i,input in enumerate(self.inputFormats): | |
| 352 xdict['inputs' ]+='<param name="input{0}" type="data" format="{1}" label="Select a suitable input file from your history"/> \n'.format(i+1, input) | |
| 353 xdict['command_inputs'] += ' $input{0}'.format(i+1) | |
| 354 else: | |
| 355 xdict['command_inputs'] = '' # assume no input - eg a random data generator | |
| 356 xdict['inputs'] = '' | |
| 357 # I find setting the job name not very logical. can be changed in workflows anyway. xdict['inputs'] += '<param name="job_name" type="text" label="Supply a name for the outputs to remind you what they contain" value="%s"/> \n' % self.toolname | |
| 358 xdict['toolname'] = self.toolname | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
359 xdict['toolname_sanitized'] = self.toolname_sanitized |
| 0 | 360 xdict['toolid'] = self.toolid |
| 361 xdict['interpreter'] = self.opts.interpreter | |
| 362 xdict['scriptname'] = self.sfile | |
| 363 if self.opts.make_HTML: | |
| 364 xdict['command_outputs'] += ' --output_dir "$html_file.files_path" --output_html "$html_file" --make_HTML "yes"' | |
| 365 xdict['outputs'] += ' <data format="html" name="html_file"/>\n' | |
| 366 else: | |
| 367 xdict['command_outputs'] += ' --output_dir "./"' | |
| 368 #print self.opts.output_tab | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
369 if self.opts.output_tab!="None": |
| 0 | 370 xdict['command_outputs'] += ' --output_tab "$tab_file"' |
| 371 xdict['outputs'] += ' <data format="%s" name="tab_file"/>\n' % self.outFormats | |
| 372 xdict['command'] = newCommand % xdict | |
| 373 #print xdict['outputs'] | |
| 374 xmls = newXML % xdict | |
| 375 xf = open(self.xmlfile,'w') | |
| 376 xf.write(xmls) | |
| 377 xf.write('\n') | |
| 378 xf.close() | |
| 379 # ready for the tarball | |
| 380 | |
| 381 | |
| 382 def makeTooltar(self): | |
| 383 """ | |
| 384 a tool is a gz tarball with eg | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
385 /toolname_sanitized/tool.xml /toolname_sanitized/tool.py /toolname_sanitized/test-data/test1_in.foo ... |
| 0 | 386 """ |
| 387 retval = self.run() | |
| 388 if retval: | |
| 389 print >> sys.stderr,'## Run failed. Cannot build yet. Please fix and retry' | |
| 390 sys.exit(1) | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
391 tdir = self.toolname_sanitized |
| 0 | 392 os.mkdir(tdir) |
| 393 self.makeXML() | |
| 394 if self.opts.make_HTML: | |
| 395 if self.opts.help_text: | |
| 396 hlp = open(self.opts.help_text,'r').read() | |
| 397 else: | |
| 398 hlp = 'Please ask the tool author for help as none was supplied at tool generation\n' | |
| 399 if self.opts.include_dependencies: | |
| 400 tooldepcontent = toolhtmldepskel % hlp | |
| 401 depf = open(os.path.join(tdir,'tool_dependencies.xml'),'w') | |
| 402 depf.write(tooldepcontent) | |
| 403 depf.write('\n') | |
| 404 depf.close() | |
| 405 if self.opts.input_tab <> 'None': # no reproducible test otherwise? TODO: maybe.. | |
| 406 testdir = os.path.join(tdir,'test-data') | |
| 407 os.mkdir(testdir) # make tests directory | |
| 408 for i in self.opts.input_tab: | |
| 409 #print i | |
| 410 shutil.copyfile(i,os.path.join(testdir,self.test1Input)) | |
| 411 if not self.opts.output_tab: | |
| 412 shutil.copyfile(self.opts.output_tab,os.path.join(testdir,self.test1Output)) | |
| 413 if self.opts.make_HTML: | |
| 414 shutil.copyfile(self.opts.output_html,os.path.join(testdir,self.test1HTML)) | |
| 415 if self.opts.output_dir: | |
| 416 shutil.copyfile(self.tlog,os.path.join(testdir,'test1_out.log')) | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
417 outpif = '%s.py' % self.toolname_sanitized # new name |
| 0 | 418 outpiname = os.path.join(tdir,outpif) # path for the tool tarball |
| 419 pyin = os.path.basename(self.pyfile) # our name - we rewrite ourselves (TM) | |
| 420 notes = ['# %s - a self annotated version of %s generated by running %s\n' % (outpiname,pyin,pyin),] | |
| 421 notes.append('# to make a new Galaxy tool called %s\n' % self.toolname) | |
| 422 notes.append('# User %s at %s\n' % (self.opts.user_email,timenow())) | |
| 423 pi=[line.replace('if opts.dockerized==0:', 'if False:') for line in open(self.pyfile)] #do not run docker in the generated tool | |
| 424 notes += pi | |
| 425 outpi = open(outpiname,'w') | |
| 426 outpi.write(''.join(notes)) | |
| 427 outpi.write('\n') | |
| 428 outpi.close() | |
| 429 stname = os.path.join(tdir,self.sfile) | |
| 430 if not os.path.exists(stname): | |
| 431 shutil.copyfile(self.sfile, stname) | |
| 432 xtname = os.path.join(tdir,self.xmlfile) | |
| 433 if not os.path.exists(xtname): | |
| 434 shutil.copyfile(self.xmlfile,xtname) | |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
435 tarpath = "%s.gz" % self.toolname_sanitized |
| 0 | 436 tar = tarfile.open(tarpath, "w:gz") |
|
4
de4889098f45
Allow whitespace in gene name and fix missing creation of output section.
mvdbeek
parents:
2
diff
changeset
|
437 tar.add(tdir,arcname=self.toolname_sanitized) |
| 0 | 438 tar.close() |
| 439 shutil.copyfile(tarpath,self.opts.new_tool) | |
| 440 shutil.rmtree(tdir) | |
| 441 ## TODO: replace with optional direct upload to local toolshed? | |
| 442 return retval | |
| 443 | |
| 444 | |
| 445 def compressPDF(self,inpdf=None,thumbformat='png'): | |
| 446 """need absolute path to pdf | |
| 447 note that GS gets confoozled if no $TMP or $TEMP | |
| 448 so we set it | |
| 449 """ | |
| 450 assert os.path.isfile(inpdf), "## Input %s supplied to %s compressPDF not found" % (inpdf,self.myName) | |
| 451 hlog = os.path.join(self.opts.output_dir,"compress_%s.txt" % os.path.basename(inpdf)) | |
| 452 sto = open(hlog,'a') | |
| 453 our_env = os.environ.copy() | |
| 454 our_tmp = our_env.get('TMP',None) | |
| 455 if not our_tmp: | |
| 456 our_tmp = our_env.get('TEMP',None) | |
| 457 if not (our_tmp and os.path.exists(our_tmp)): | |
| 458 newtmp = os.path.join(self.opts.output_dir,'tmp') | |
| 459 try: | |
| 460 os.mkdir(newtmp) | |
| 461 except: | |
| 462 sto.write('## WARNING - cannot make %s - it may exist or permissions need fixing\n' % newtmp) | |
| 463 our_env['TEMP'] = newtmp | |
| 464 if not self.temp_warned: | |
| 465 sto.write('## WARNING - no $TMP or $TEMP!!! Please fix - using %s temporarily\n' % newtmp) | |
| 466 self.temp_warned = True | |
| 467 outpdf = '%s_compressed' % inpdf | |
| 468 cl = ["gs", "-sDEVICE=pdfwrite", "-dNOPAUSE", "-dUseCIEColor", "-dBATCH","-dPDFSETTINGS=/printer", "-sOutputFile=%s" % outpdf,inpdf] | |
| 469 x = subprocess.Popen(cl,stdout=sto,stderr=sto,cwd=self.opts.output_dir,env=our_env) | |
| 470 retval1 = x.wait() | |
| 471 sto.close() | |
| 472 if retval1 == 0: | |
| 473 os.unlink(inpdf) | |
| 474 shutil.move(outpdf,inpdf) | |
| 475 os.unlink(hlog) | |
| 476 hlog = os.path.join(self.opts.output_dir,"thumbnail_%s.txt" % os.path.basename(inpdf)) | |
| 477 sto = open(hlog,'w') | |
| 478 outpng = '%s.%s' % (os.path.splitext(inpdf)[0],thumbformat) | |
| 479 if self.useGM: | |
| 480 cl2 = ['gm', 'convert', inpdf, outpng] | |
| 481 else: # assume imagemagick | |
| 482 cl2 = ['convert', inpdf, outpng] | |
| 483 x = subprocess.Popen(cl2,stdout=sto,stderr=sto,cwd=self.opts.output_dir,env=our_env) | |
| 484 retval2 = x.wait() | |
| 485 sto.close() | |
| 486 if retval2 == 0: | |
| 487 os.unlink(hlog) | |
| 488 retval = retval1 or retval2 | |
| 489 return retval | |
| 490 | |
| 491 | |
| 492 def getfSize(self,fpath,outpath): | |
| 493 """ | |
| 494 format a nice file size string | |
| 495 """ | |
| 496 size = '' | |
| 497 fp = os.path.join(outpath,fpath) | |
| 498 if os.path.isfile(fp): | |
| 499 size = '0 B' | |
| 500 n = float(os.path.getsize(fp)) | |
| 501 if n > 2**20: | |
| 502 size = '%1.1f MB' % (n/2**20) | |
| 503 elif n > 2**10: | |
| 504 size = '%1.1f KB' % (n/2**10) | |
| 505 elif n > 0: | |
| 506 size = '%d B' % (int(n)) | |
| 507 return size | |
| 508 | |
| 509 def makeHtml(self): | |
| 510 """ Create an HTML file content to list all the artifacts found in the output_dir | |
| 511 """ | |
| 512 | |
| 513 galhtmlprefix = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> | |
| 514 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en"> | |
| 515 <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> | |
| 516 <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" /> | |
| 517 <title></title> | |
| 518 <link rel="stylesheet" href="/static/style/base.css" type="text/css" /> | |
| 519 </head> | |
| 520 <body> | |
| 521 <div class="toolFormBody"> | |
| 522 """ | |
| 523 galhtmlattr = """<hr/><div class="infomessage">This tool (%s) was generated by the <a href="https://bitbucket.org/fubar/galaxytoolfactory/overview">Galaxy Tool Factory</a></div><br/>""" | |
| 524 galhtmlpostfix = """</div></body></html>\n""" | |
| 525 | |
| 526 flist = os.listdir(self.opts.output_dir) | |
| 527 flist = [x for x in flist if x <> 'Rplots.pdf'] | |
| 528 flist.sort() | |
| 529 html = [] | |
| 530 html.append(galhtmlprefix % progname) | |
| 531 html.append('<div class="infomessage">Galaxy Tool "%s" run at %s</div><br/>' % (self.toolname,timenow())) | |
| 532 fhtml = [] | |
| 533 if len(flist) > 0: | |
| 534 logfiles = [x for x in flist if x.lower().endswith('.log')] # log file names determine sections | |
| 535 logfiles.sort() | |
| 536 logfiles = [x for x in logfiles if abspath(x) <> abspath(self.tlog)] | |
| 537 logfiles.append(abspath(self.tlog)) # make it the last one | |
| 538 pdflist = [] | |
| 539 npdf = len([x for x in flist if os.path.splitext(x)[-1].lower() == '.pdf']) | |
| 540 for rownum,fname in enumerate(flist): | |
| 541 dname,e = os.path.splitext(fname) | |
| 542 sfsize = self.getfSize(fname,self.opts.output_dir) | |
| 543 if e.lower() == '.pdf' : # compress and make a thumbnail | |
| 544 thumb = '%s.%s' % (dname,self.thumbformat) | |
| 545 pdff = os.path.join(self.opts.output_dir,fname) | |
| 546 retval = self.compressPDF(inpdf=pdff,thumbformat=self.thumbformat) | |
| 547 if retval == 0: | |
| 548 pdflist.append((fname,thumb)) | |
| 549 else: | |
| 550 pdflist.append((fname,fname)) | |
| 551 if (rownum+1) % 2 == 0: | |
| 552 fhtml.append('<tr class="odd_row"><td><a href="%s">%s</a></td><td>%s</td></tr>' % (fname,fname,sfsize)) | |
| 553 else: | |
| 554 fhtml.append('<tr><td><a href="%s">%s</a></td><td>%s</td></tr>' % (fname,fname,sfsize)) | |
| 555 for logfname in logfiles: # expect at least tlog - if more | |
| 556 if abspath(logfname) == abspath(self.tlog): # handled later | |
| 557 sectionname = 'All tool run' | |
| 558 if (len(logfiles) > 1): | |
| 559 sectionname = 'Other' | |
| 560 ourpdfs = pdflist | |
| 561 else: | |
| 562 realname = os.path.basename(logfname) | |
| 563 sectionname = os.path.splitext(realname)[0].split('_')[0] # break in case _ added to log | |
| 564 ourpdfs = [x for x in pdflist if os.path.basename(x[0]).split('_')[0] == sectionname] | |
| 565 pdflist = [x for x in pdflist if os.path.basename(x[0]).split('_')[0] <> sectionname] # remove | |
| 566 nacross = 1 | |
| 567 npdf = len(ourpdfs) | |
| 568 | |
| 569 if npdf > 0: | |
| 570 nacross = math.sqrt(npdf) ## int(round(math.log(npdf,2))) | |
| 571 if int(nacross)**2 != npdf: | |
| 572 nacross += 1 | |
| 573 nacross = int(nacross) | |
| 574 width = min(400,int(1200/nacross)) | |
| 575 html.append('<div class="toolFormTitle">%s images and outputs</div>' % sectionname) | |
| 576 html.append('(Click on a thumbnail image to download the corresponding original PDF image)<br/>') | |
| 577 ntogo = nacross # counter for table row padding with empty cells | |
| 578 html.append('<div><table class="simple" cellpadding="2" cellspacing="2">\n<tr>') | |
| 579 for i,paths in enumerate(ourpdfs): | |
| 580 fname,thumb = paths | |
| 581 s= """<td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="5" width="%d" | |
| 582 alt="Image called %s"/></a></td>\n""" % (fname,thumb,fname,width,fname) | |
| 583 if ((i+1) % nacross == 0): | |
| 584 s += '</tr>\n' | |
| 585 ntogo = 0 | |
| 586 if i < (npdf - 1): # more to come | |
| 587 s += '<tr>' | |
| 588 ntogo = nacross | |
| 589 else: | |
| 590 ntogo -= 1 | |
| 591 html.append(s) | |
| 592 if html[-1].strip().endswith('</tr>'): | |
| 593 html.append('</table></div>\n') | |
| 594 else: | |
| 595 if ntogo > 0: # pad | |
| 596 html.append('<td> </td>'*ntogo) | |
| 597 html.append('</tr></table></div>\n') | |
| 598 logt = open(logfname,'r').readlines() | |
| 599 logtext = [x for x in logt if x.strip() > ''] | |
| 600 html.append('<div class="toolFormTitle">%s log output</div>' % sectionname) | |
| 601 if len(logtext) > 1: | |
| 602 html.append('\n<pre>\n') | |
| 603 html += logtext | |
| 604 html.append('\n</pre>\n') | |
| 605 else: | |
| 606 html.append('%s is empty<br/>' % logfname) | |
| 607 if len(fhtml) > 0: | |
| 608 fhtml.insert(0,'<div><table class="colored" cellpadding="3" cellspacing="3"><tr><th>Output File Name (click to view)</th><th>Size</th></tr>\n') | |
| 609 fhtml.append('</table></div><br/>') | |
| 610 html.append('<div class="toolFormTitle">All output files available for downloading</div>\n') | |
| 611 html += fhtml # add all non-pdf files to the end of the display | |
| 612 else: | |
| 613 html.append('<div class="warningmessagelarge">### Error - %s returned no files - please confirm that parameters are sane</div>' % self.opts.interpreter) | |
| 614 html.append(galhtmlpostfix) | |
| 615 htmlf = file(self.opts.output_html,'w') | |
| 616 htmlf.write('\n'.join(html)) | |
| 617 htmlf.write('\n') | |
| 618 htmlf.close() | |
| 619 self.html = html | |
| 620 | |
| 621 | |
| 622 def run(self): | |
| 623 """ | |
| 624 scripts must be small enough not to fill the pipe! | |
| 625 """ | |
| 626 if self.treatbashSpecial and self.opts.interpreter in ['bash','sh']: | |
| 627 retval = self.runBash() | |
| 628 else: | |
| 629 if self.opts.output_dir: | |
| 630 ste = open(self.elog,'w') | |
| 631 sto = open(self.tlog,'w') | |
| 632 sto.write('## Toolfactory generated command line = %s\n' % ' '.join(self.cl)) | |
| 633 sto.flush() | |
| 634 p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=ste,stdin=subprocess.PIPE,cwd=self.opts.output_dir) | |
| 635 else: | |
| 636 p = subprocess.Popen(self.cl,shell=False,stdin=subprocess.PIPE) | |
| 637 p.stdin.write(self.script) | |
| 638 p.stdin.close() | |
| 639 retval = p.wait() | |
| 640 if self.opts.output_dir: | |
| 641 sto.close() | |
| 642 ste.close() | |
| 643 err = open(self.elog,'r').readlines() | |
| 644 if retval <> 0 and err: # problem | |
| 645 print >> sys.stderr,err #same problem, need to capture docker stdin/stdout | |
| 646 if self.opts.make_HTML: | |
| 647 self.makeHtml() | |
| 648 return retval | |
| 649 | |
| 650 def runBash(self): | |
| 651 """ | |
| 652 cannot use - for bash so use self.sfile | |
| 653 """ | |
| 654 if self.opts.output_dir: | |
| 655 s = '## Toolfactory generated command line = %s\n' % ' '.join(self.cl) | |
| 656 sto = open(self.tlog,'w') | |
| 657 sto.write(s) | |
| 658 sto.flush() | |
| 659 p = subprocess.Popen(self.cl,shell=False,stdout=sto,stderr=sto,cwd=self.opts.output_dir) | |
| 660 else: | |
| 661 p = subprocess.Popen(self.cl,shell=False) | |
| 662 retval = p.wait() | |
| 663 if self.opts.output_dir: | |
| 664 sto.close() | |
| 665 if self.opts.make_HTML: | |
| 666 self.makeHtml() | |
| 667 return retval | |
| 668 | |
| 669 | |
| 670 def main(): | |
| 671 u = """ | |
| 672 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as: | |
| 673 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript" | |
| 674 </command> | |
| 675 """ | |
| 676 op = argparse.ArgumentParser() | |
| 677 a = op.add_argument | |
| 678 a('--script_path',default=None) | |
| 679 a('--tool_name',default=None) | |
| 680 a('--interpreter',default=None) | |
| 681 a('--output_dir',default='./') | |
| 682 a('--output_html',default=None) | |
| 683 a('--input_tab',default='None', nargs='*') | |
| 684 a('--output_tab',default='None') | |
| 685 a('--user_email',default='Unknown') | |
| 686 a('--bad_user',default=None) | |
| 687 a('--make_Tool',default=None) | |
| 688 a('--make_HTML',default=None) | |
| 689 a('--help_text',default=None) | |
| 690 a('--tool_desc',default=None) | |
| 691 a('--new_tool',default=None) | |
| 692 a('--tool_version',default=None) | |
| 693 a('--include_dependencies',default=None) | |
| 694 a('--dockerized',default=0) | |
| 695 a('--output_format', default='tabular') | |
| 696 a('--input_format', dest='input_formats', action='append', default=[]) | |
| 697 a('--additional_parameters', dest='additional_parameters', action='append', default=[]) | |
| 698 opts = op.parse_args() | |
| 699 assert not opts.bad_user,'UNAUTHORISED: %s is NOT authorized to use this tool until Galaxy admin adds %s to admin_users in universe_wsgi.ini' % (opts.bad_user,opts.bad_user) | |
| 700 assert opts.tool_name,'## Tool Factory expects a tool name - eg --tool_name=DESeq' | |
| 701 assert opts.interpreter,'## Tool Factory wrapper expects an interpreter - eg --interpreter=Rscript' | |
| 702 assert os.path.isfile(opts.script_path),'## Tool Factory wrapper expects a script path - eg --script_path=foo.R' | |
| 703 if opts.output_dir: | |
| 704 try: | |
| 705 os.makedirs(opts.output_dir) | |
| 706 except: | |
| 707 pass | |
| 708 if opts.dockerized==0: | |
| 709 switch_to_docker(opts) | |
| 710 return | |
| 711 r = ScriptRunner(opts) | |
| 712 if opts.make_Tool: | |
| 713 retcode = r.makeTooltar() | |
| 714 else: | |
| 715 retcode = r.run() | |
| 716 os.unlink(r.sfile) | |
| 717 if retcode: | |
| 718 sys.exit(retcode) # indicate failure to job runner | |
| 719 | |
| 720 | |
| 721 if __name__ == "__main__": | |
| 722 main() | |
| 723 | |
| 724 |