Mercurial > repos > fubar > brokenandnotdeletablebyowneroradmin

comparison rgToolFactory.py @ 7:7221619caefa

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Updated name and added crude gzip generator for toolshed TODO: add tests and new XML tool descriptor as soon as Greg has it nailed down.
author ross lazarus ross.lazarus@gmail.com
date Sat, 02 Jun 2012 10:43:08 +1000
parents
children 220885b2d7ee
comparison
equal deleted inserted replaced
6:78044a3d4a21 7:7221619caefa
1 # rgDynamicScriptWrapper.py
2 # derived from
3 # rgBaseScriptWrapper.py
4 # to run some user supplied code
5 # extremely dangerous
6 # trusted users only - private site only
7 # a list in the xml is searched - only users in the list can run this tool.
8 #
9 # copyright ross lazarus (ross.lazarus@gmail.com) May 2012
10 #
11 # all rights reserved
12 # Licensed under the LGPL for your pleasure
13 # Derived from rgDGE.py in May 2012
14 # generalized to run required interpreter
15 # to make your own tools based on a given script and interpreter such as perl or python
16 # clone this and the corresponding xml wrapper
17 # replace the parameters/inputs/outputs and the configfile contents with your script
18 # Use the $foo syntax to place your parameter values inside the script to assign them - at run time, the script will be used as a template
19 # and returned as part of the output to the user - with the right values for all the parameters.
20 # Note that this assumes you want all the outputs arranged as a single Html file output
21 # after this generic script runner runs your script with the specified interpreter,
22 # it will collect all output files into the specified output_html, making thumbnails for all the pdfs it finds and making links for all the other files.
23
24 import sys
25 import shutil
26 import subprocess
27 import os
28 import time
29 import tempfile
30 import optparse
31 import tarfile
32 import re
33 progname = os.path.split(sys.argv[0])[1]
34 myversion = 'V000.1 May 2012'
35 verbose = False
36 debug = False
37
38
39 galhtmlprefix = """<?xml version="1.0" encoding="utf-8" ?>
40 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
41 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
42 <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
43 <meta name="generator" content="Galaxy %s tool output - see http://g2.trac.bx.psu.edu/" />
44 <title></title>
45 <link rel="stylesheet" href="/static/style/base.css" type="text/css" />
46 </head>
47 <body>
48 <div class="document">
49 """
50 galhtmlattr = """<b><a href="http://rgenetics.org">Galaxy Rgenetics Base Script Wrapper based </a> tool output %s run at %s</b><br/>"""
51 galhtmlpostfix = """</div></body></html>\n"""
52
53 def timenow():
54 """return current time as a string
55 """
56 return time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(time.time()))
57 # characters that are allowed but need to be escaped
58 mapped_chars = { '>' :'__gt__',
59 '<' :'__lt__',
60 "'" :'__sq__',
61 '"' :'__dq__',
62 '{' :'__oc__',
63 '}' :'__cc__',
64 '@' : '__at__',
65 '\n' : '__cn__',
66 '\r' : '__cr__',
67 '\t' : '__tc__',
68 '#' : '__pd__',
69 '[' :'__ob__',
70 ']' :'__cb__',
71 '\t' : 'Xt',
72 'systemCallsAreNotAllowed' : 'system'
73 }
74
75 def restore_text(text):
76 """Restores sanitized text"""
77 if not text:
78 return text
79 for key, value in mapped_chars.items():
80 text = text.replace(value, key)
81 return text
82
83 class ScriptRunner:
84 """class is a wrapper for an arbitrary script
85 """
86
87 def __init__(self,opts=None):
88 """
89 run the script
90 cheetah/galaxy will provide an escaped string so
91 __pd__ your script goes here
92 __cr____cn__ourargs __lt__- commandArgs(TRUE)
93 __cr____cn__inf = ourargs[1]
94 __cr____cn__outf = ourargs[2]
95 __cr____cn__inp = read.table(inf,head=T,rownames=F,sep=__sq__Xt__sq__)
96 __cr____cn__ write.table(inp,outf, quote=FALSE, sep=__dq__Xt__dq__,row.names=F)
97 __cr____cn__sessionInfo()
98 __cr____cn__
99 """
100 self.myname = sys.argv[0] # get our name because we write ourselves out as a tool later
101 self.thumbformat = 'jpg'
102 self.opts = opts
103 self.toolname = re.sub('[^a-zA-Z0-9_]+', '', opts.tool_name)
104 s = open(self.opts.script_path,'r').read()
105 self.script = restore_text(s)
106 self.pyfile = self.myname
107 self.xmlfile = '%s.xml' % os.path.splitext(self.pyfile)[0] # punt
108 self.sfile = '%s.%s' % (self.toolname,opts.interpreter)
109 localscript = open(self.sfile,'w')
110 localscript.write(self.script)
111 localscript.close()
112 if opts.output_dir or self.opts.makeTool: # may not want these complexities if a simple script
113 self.tlog = os.path.join(opts.output_dir,"%s_runner.log" % self.toolname)
114 artifactpath = os.path.join(opts.output_dir,'%s_run.script' % self.toolname)
115 artifact = open(artifactpath,'w')
116 artifact.write(self.script)
117 artifact.write('\n')
118 artifact.close()
119 self.cl = []
120 self.html = []
121 a = self.cl.append
122 a(opts.interpreter)
123 a('-') # use stdin
124 a(opts.input_tab)
125 a(opts.output_tab)
126
127 def makeTooltar(self):
128 """
129 a tool is a gz tarball with eg
130 /toolname/tool.xml /toolname/tool.py /toolname/test-data/test1_in.foo ...
131 """
132 retval = self.run()
133 if retval:
134 print >> sys.stderr,'## Run failed. Cannot build yet. Please fix and retry'
135 sys.exit(1)
136 tarpath = os.path.join(self.opts.output_dir,"%s.gz" % self.toolname)
137 tar = tarfile.open(tarpath, "w:gz")
138 tar.add(self.xmlfile,arcname='%s.xml' % self.toolname)
139 tar.add(self.pyfile,arcname=os.path.basename(self.pyfile))
140 tar.add(self.sfile,arcname=self.sfile)
141 tar.close()
142 self.makeHtml()
143 return retval
144
145 def compressPDF(self,inpdf=None,thumbformat='png'):
146 """need absolute path to pdf
147 """
148 assert os.path.isfile(inpdf), "## Input %s supplied to %s compressPDF not found" % (inpdf,self.myName)
149 hf,hlog = tempfile.mkstemp(suffix="%s.log" % self.toolname)
150 sto = open(hlog,'w')
151 outpdf = '%s_compressed' % inpdf
152 cl = ["gs", "-sDEVICE=pdfwrite", "-dNOPAUSE", "-dBATCH", "-sOutputFile=%s" % outpdf,inpdf]
153 x = subprocess.Popen(cl,stdout=sto,stderr=sto,cwd=self.opts.output_dir)
154 retval1 = x.wait()
155 if retval1 == 0:
156 os.unlink(inpdf)
157 shutil.move(outpdf,inpdf)
158 outpng = '%s.%s' % (os.path.splitext(inpdf)[0],thumbformat)
159 cl2 = ['convert', inpdf, outpng]
160 x = subprocess.Popen(cl2,stdout=sto,stderr=sto,cwd=self.opts.output_dir)
161 retval2 = x.wait()
162 sto.close()
163 retval = retval1 or retval2
164 return retval
165
166
167 def getfSize(self,fpath,outpath):
168 """
169 format a nice file size string
170 """
171 size = ''
172 fp = os.path.join(outpath,fpath)
173 if os.path.isfile(fp):
174 n = float(os.path.getsize(fp))
175 if n > 2**20:
176 size = ' (%1.1f MB)' % (n/2**20)
177 elif n > 2**10:
178 size = ' (%1.1f KB)' % (n/2**10)
179 elif n > 0:
180 size = ' (%d B)' % (int(n))
181 return size
182
183 def makeHtml(self):
184 """
185 """
186 flist = os.listdir(self.opts.output_dir)
187 flist = [x for x in flist if x <> 'Rplots.pdf']
188 flist.sort()
189 html = [galhtmlprefix % progname,]
190 html.append('<h2>Galaxy %s outputs run at %s</h2><br/>\n' % (self.toolname,timenow()))
191 fhtml = []
192 if len(flist) > 0:
193 html.append('<table cellpadding="3" cellspacing="3">\n')
194 for fname in flist:
195 dname,e = os.path.splitext(fname)
196 sfsize = self.getfSize(fname,self.opts.output_dir)
197 if e.lower() == '.pdf' : # compress and make a thumbnail
198 thumb = '%s.%s' % (dname,self.thumbformat)
199 pdff = os.path.join(self.opts.output_dir,fname)
200 retval = self.compressPDF(inpdf=pdff,thumbformat=self.thumbformat)
201 if retval == 0:
202 s= '<tr><td><a href="%s"><img src="%s" title="Click to download a PDF of %s" hspace="10" width="600"></a></td></tr>\n' % (fname,thumb,fname)
203 html.append(s)
204 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize))
205 else:
206 fhtml.append('<li><a href="%s">%s %s</a></li>' % (fname,fname,sfsize))
207 html.append('</table>\n')
208 if len(fhtml) > 0:
209 fhtml.insert(0,'<ul>')
210 fhtml.append('</ul>')
211 html += fhtml # add all non-pdf files to the end of the display
212 else:
213 html.append('<h2>### Error - %s returned no files - please confirm that parameters are sane</h1>' % self.opts.interpreter)
214 html.append('<h3>%s log follows below</h3><hr><pre>\n' % self.opts.interpreter)
215 rlog = open(self.tlog,'r').readlines()
216 html += rlog
217 html.append('%s CL = %s</br>\n' % (self.toolname,' '.join(sys.argv)))
218 html.append('CL = %s</br>\n' % (' '.join(self.cl)))
219 html.append('</pre>\n')
220 html.append(galhtmlattr % (progname,timenow()))
221 html.append(galhtmlpostfix)
222 htmlf = file(self.opts.output_html,'w')
223 htmlf.write('\n'.join(html))
224 htmlf.write('\n')
225 htmlf.close()
226 self.html = html
227
228
229 def run(self):
230 """
231 """
232 if self.opts.output_dir or self.opts.makeTool:
233 sto = open(self.tlog,'w')
234 p = subprocess.Popen(' '.join(self.cl),shell=True,stdout=sto,stderr=sto,stdin=subprocess.PIPE,cwd=self.opts.output_dir)
235 else:
236 p = subprocess.Popen(' '.join(self.cl),shell=True,stdin=subprocess.PIPE)
237 p.stdin.write(self.script)
238 p.stdin.close()
239 retval = p.wait()
240 if self.opts.output_dir or self.opts.makeTool:
241 sto.close()
242 self.makeHtml()
243 return retval
244
245
246 def main():
247 u = """
248 This is a Galaxy wrapper. It expects to be called by a special purpose tool.xml as:
249 <command interpreter="python">rgBaseScriptWrapper.py --script_path "$scriptPath" --tool_name "foo" --interpreter "Rscript"
250 </command>
251 """
252 op = optparse.OptionParser()
253 a = op.add_option
254 a('--script_path',default=None)
255 a('--tool_name',default=None)
256 a('--interpreter',default=None)
257 a('--output_dir',default=None)
258 a('--output_html',default=None)
259 a('--input_tab',default='NONE')
260 a('--output_tab',default='NONE')
261 a('--user_email',default=None)
262 a('--bad_user',default=None)
263 a('--makeTool',default=None)
264 opts, args = op.parse_args()
265 assert not opts.bad_user,'%s is NOT authorized to use this tool. Please ask your friendly admin' % opts.bad_user
266 assert opts.tool_name,'## Tool Factory expects a tool name - eg --tool_name=DESeq'
267 assert opts.interpreter,'## Tool Factory wrapper expects an interpreter - eg --interpreter=Rscript'
268 assert os.path.isfile(opts.script_path),'## Tool Factory wrapper expects a script path - eg --script_path=foo.R'
269 if opts.output_dir:
270 try:
271 os.makedirs(opts.output_dir)
272 except:
273 pass
274 r = ScriptRunner(opts)
275 if opts.makeTool:
276 retcode = r.makeTooltar()
277 else:
278 retcode = r.run()
279 if retcode:
280 sys.exit(retcode) # indicate failure to job runner
281
282
283 if __name__ == "__main__":
284 main()
285
286