Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/planemo/galaxy_config.py @ 0:d67268158946 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b
author bcclaywell
date Mon, 12 Oct 2015 17:43:33 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d67268158946
1 from __future__ import absolute_import
2 from __future__ import print_function
3
4 import contextlib
5 import os
6 import random
7 import shutil
8 import time
9 from six.moves.urllib.request import urlopen
10 from six import iteritems
11 from string import Template
12 from tempfile import mkdtemp
13 from six.moves.urllib.request import urlretrieve
14
15 import click
16
17 from planemo import galaxy_run
18 from planemo.io import warn
19 from planemo.io import shell
20 from planemo.io import write_file
21 from planemo.io import kill_pid_file
22 from planemo import git
23 from planemo.shed import tool_shed_url
24 from planemo.bioblend import (
25 galaxy,
26 ensure_module,
27 )
28
29
30 NO_TEST_DATA_MESSAGE = (
31 "planemo couldn't find a target test-data directory, you should likely "
32 "create a test-data directory or pass an explicit path using --test_data."
33 )
34
35 WEB_SERVER_CONFIG_TEMPLATE = """
36 [server:${server_name}]
37 use = egg:Paste#http
38 port = ${port}
39 host = ${host}
40 use_threadpool = True
41 threadpool_kill_thread_limit = 10800
42 [app:main]
43 paste.app_factory = galaxy.web.buildapp:app_factory
44 """
45
46 TOOL_CONF_TEMPLATE = """<toolbox>
47 <tool file="data_source/upload.xml" />
48 ${tool_definition}
49 </toolbox>
50 """
51
52 SHED_TOOL_CONF_TEMPLATE = """<?xml version="1.0"?>
53 <toolbox tool_path="${shed_tools_path}">
54 </toolbox>
55 """
56
57
58 EMPTY_JOB_METRICS_TEMPLATE = """<?xml version="1.0"?>
59 <job_metrics>
60 </job_metrics>
61 """
62
63
64 BREW_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
65 <homebrew />
66 <!--
67 <homebrew versionless="true" />
68 -->
69 </dependency_resolvers>
70 """
71
72 SHED_DEPENDENCY_RESOLUTION_CONF = """<dependency_resolvers>
73 <tool_shed_tap />
74 </dependency_resolvers>
75 """
76
77 TOOL_SHEDS_CONF = """<tool_sheds>
78 <tool_shed name="Target Shed" url="${shed_target_url}" />
79 </tool_sheds>
80 """
81
82 # Provide some shortcuts for simple/common dependency resolutions strategies.
83 STOCK_DEPENDENCY_RESOLUTION_STRATEGIES = {
84 "brew_dependency_resolution": BREW_DEPENDENCY_RESOLUTION_CONF,
85 "shed_dependency_resolution": SHED_DEPENDENCY_RESOLUTION_CONF,
86 }
87
88 EMPTY_TOOL_CONF_TEMPLATE = """<toolbox></toolbox>"""
89
90 DOWNLOADS_URL = ("https://raw.githubusercontent.com/"
91 "jmchilton/galaxy-downloads/master/")
92 DOWNLOADABLE_MIGRATION_VERSIONS = [127, 120, 117]
93 LATEST_URL = DOWNLOADS_URL + "latest.sqlite"
94
95 FAILED_TO_FIND_GALAXY_EXCEPTION = (
96 "Failed to find Galaxy root directory - please explicitly specify one "
97 "with --galaxy_root."
98 )
99
100
101 @contextlib.contextmanager
102 def galaxy_config(ctx, tool_paths, for_tests=False, **kwds):
103 test_data_dir = _find_test_data(tool_paths, **kwds)
104 tool_data_table = _find_tool_data_table(
105 tool_paths,
106 test_data_dir=test_data_dir,
107 **kwds
108 )
109 galaxy_root = _check_galaxy(ctx, **kwds)
110 install_galaxy = galaxy_root is None
111 config_directory = kwds.get("config_directory", None)
112
113 def config_join(*args):
114 return os.path.join(config_directory, *args)
115
116 created_config_directory = False
117 if not config_directory:
118 created_config_directory = True
119 config_directory = mkdtemp()
120 try:
121 latest_galaxy = False
122 if install_galaxy:
123 _install_galaxy(ctx, config_directory, kwds)
124 latest_galaxy = True
125 galaxy_root = config_join("galaxy-dev")
126
127 _handle_dependency_resolution(config_directory, kwds)
128 _handle_job_metrics(config_directory, kwds)
129 tool_definition = _tool_conf_entry_for(tool_paths)
130 empty_tool_conf = config_join("empty_tool_conf.xml")
131 shed_tool_conf = _shed_tool_conf(install_galaxy, config_directory)
132 tool_conf = config_join("tool_conf.xml")
133 database_location = config_join("galaxy.sqlite")
134 shed_tools_path = config_join("shed_tools")
135 sheds_config_path = _configure_sheds_config_file(
136 ctx, config_directory, **kwds
137 )
138 preseeded_database = True
139 master_api_key = kwds.get("master_api_key", "test_key")
140 dependency_dir = os.path.join(config_directory, "deps")
141
142 try:
143 _download_database_template(
144 galaxy_root,
145 database_location,
146 latest=latest_galaxy
147 )
148 except Exception as e:
149 print(e)
150 # No network access - just roll forward from null.
151 preseeded_database = False
152
153 os.makedirs(shed_tools_path)
154 server_name = "planemo%d" % random.randint(0, 100000)
155 port = kwds.get("port", 9090)
156 template_args = dict(
157 port=port,
158 host=kwds.get("host", "127.0.0.1"),
159 server_name=server_name,
160 temp_directory=config_directory,
161 shed_tools_path=shed_tools_path,
162 database_location=database_location,
163 tool_definition=tool_definition,
164 tool_conf=tool_conf,
165 debug=kwds.get("debug", "true"),
166 master_api_key=master_api_key,
167 id_secret=kwds.get("id_secret", "test_secret"),
168 log_level=kwds.get("log_level", "DEBUG"),
169 )
170 tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
171 properties = dict(
172 tool_dependency_dir=dependency_dir,
173 file_path="${temp_directory}/files",
174 new_file_path="${temp_directory}/tmp",
175 tool_config_file=tool_config_file,
176 tool_sheds_config_file=sheds_config_path,
177 check_migrate_tools="False",
178 manage_dependency_relationships="False",
179 job_working_directory="${temp_directory}/job_working_directory",
180 template_cache_path="${temp_directory}/compiled_templates",
181 citation_cache_type="file",
182 citation_cache_data_dir="${temp_directory}/citations/data",
183 citation_cache_lock_dir="${temp_directory}/citations/lock",
184 collect_outputs_from="job_working_directory",
185 database_auto_migrate="True",
186 cleanup_job="never",
187 master_api_key="${master_api_key}",
188 id_secret="${id_secret}",
189 log_level="${log_level}",
190 debug="${debug}",
191 watch_tools="auto",
192 tool_data_table_config_path=tool_data_table,
193 integrated_tool_panel_config=("${temp_directory}/"
194 "integrated_tool_panel_conf.xml"),
195 # Use in-memory database for kombu to avoid database contention
196 # during tests.
197 amqp_internal_connection="sqlalchemy+sqlite://",
198 migrated_tools_config=empty_tool_conf,
199 test_data_dir=test_data_dir, # TODO: make gx respect this
200 )
201 if not for_tests:
202 properties["database_connection"] = \
203 "sqlite:///${database_location}?isolation_level=IMMEDIATE"
204
205 _handle_kwd_overrides(properties, kwds)
206
207 # TODO: consider following property
208 # watch_tool = False
209 # datatypes_config_file = config/datatypes_conf.xml
210 # welcome_url = /static/welcome.html
211 # logo_url = /
212 # sanitize_all_html = True
213 # serve_xss_vulnerable_mimetypes = False
214 # track_jobs_in_database = None
215 # outputs_to_working_directory = False
216 # retry_job_output_collection = 0
217
218 env = _build_env_for_galaxy(properties, template_args)
219 if install_galaxy:
220 _build_eggs_cache(ctx, env, kwds)
221 _build_test_env(properties, env)
222 env['GALAXY_TEST_SHED_TOOL_CONF'] = shed_tool_conf
223
224 # No need to download twice - would GALAXY_TEST_DATABASE_CONNECTION
225 # work?
226 if preseeded_database:
227 env["GALAXY_TEST_DB_TEMPLATE"] = os.path.abspath(database_location)
228 env["GALAXY_TEST_UPLOAD_ASYNC"] = "false"
229 env["GALAXY_DEVELOPMENT_ENVIRONMENT"] = "1"
230 web_config = _sub(WEB_SERVER_CONFIG_TEMPLATE, template_args)
231 write_file(config_join("galaxy.ini"), web_config)
232 tool_conf_contents = _sub(TOOL_CONF_TEMPLATE, template_args)
233 write_file(tool_conf, tool_conf_contents)
234 write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)
235
236 shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
237 write_file(shed_tool_conf, shed_tool_conf_contents)
238
239 yield GalaxyConfig(
240 galaxy_root,
241 config_directory,
242 env,
243 test_data_dir,
244 port,
245 server_name,
246 master_api_key,
247 )
248 finally:
249 cleanup = not kwds.get("no_cleanup", False)
250 if created_config_directory and cleanup:
251 shutil.rmtree(config_directory)
252
253
254 class GalaxyConfig(object):
255
256 def __init__(
257 self,
258 galaxy_root,
259 config_directory,
260 env,
261 test_data_dir,
262 port,
263 server_name,
264 master_api_key,
265 ):
266 self.galaxy_root = galaxy_root
267 self.config_directory = config_directory
268 self.env = env
269 self.test_data_dir = test_data_dir
270 # Runtime server configuration stuff not used if testing...
271 # better design might be GalaxyRootConfig and GalaxyServerConfig
272 # as two separate objects.
273 self.port = port
274 self.server_name = server_name
275 self.master_api_key = master_api_key
276
277 def kill(self):
278 kill_pid_file(self.pid_file)
279
280 @property
281 def pid_file(self):
282 return os.path.join(self.galaxy_root, "%s.pid" % self.server_name)
283
284 @property
285 def gi(self):
286 ensure_module(galaxy)
287 return galaxy.GalaxyInstance(
288 url="http://localhost:%d" % self.port,
289 key=self.master_api_key
290 )
291
292 def install_repo(self, *args, **kwds):
293 self.tool_shed_client.install_repository_revision(
294 *args, **kwds
295 )
296
297 @property
298 def tool_shed_client(self):
299 return self.gi.toolShed
300
301 def wait_for_all_installed(self):
302 def status_ready(repo):
303 status = repo["status"]
304 if status in ["Installing", "New"]:
305 return False
306 if status == "Installed":
307 return True
308 raise Exception("Error installing repo status is %s" % status)
309
310 def not_ready():
311 repos = self.tool_shed_client.get_repositories()
312 return not all(map(status_ready, repos))
313
314 self._wait_for(not_ready)
315
316 # Taken from Galaxy's twilltestcase.
317 def _wait_for(self, func, **kwd):
318 sleep_amount = 0.2
319 slept = 0
320 walltime_exceeded = 1086400
321 while slept <= walltime_exceeded:
322 result = func()
323 if result:
324 time.sleep(sleep_amount)
325 slept += sleep_amount
326 sleep_amount *= 1.25
327 if slept + sleep_amount > walltime_exceeded:
328 sleep_amount = walltime_exceeded - slept
329 else:
330 break
331 assert slept < walltime_exceeded, "Action taking too long."
332
333 def cleanup(self):
334 shutil.rmtree(self.config_directory)
335
336
337 def _download_database_template(galaxy_root, database_location, latest=False):
338 if latest:
339 template_url = DOWNLOADS_URL + urlopen(LATEST_URL).read()
340 urlretrieve(template_url, database_location)
341 return True
342
343 newest_migration = _newest_migration_version(galaxy_root)
344 download_migration = None
345 for migration in DOWNLOADABLE_MIGRATION_VERSIONS:
346 if newest_migration > migration:
347 download_migration = migration
348 break
349
350 if download_migration:
351 download_name = "db_gx_rev_0%d.sqlite" % download_migration
352 download_url = DOWNLOADS_URL + download_name
353 urlretrieve(download_url, database_location)
354 return True
355 else:
356 return False
357
358
359 def _newest_migration_version(galaxy_root):
360 versions = os.path.join(galaxy_root, "lib/galaxy/model/migrate/versions")
361 version = max(map(_file_name_to_migration_version, os.listdir(versions)))
362 return version
363
364
365 def _file_name_to_migration_version(name):
366 try:
367 return int(name[0:4])
368 except ValueError:
369 return -1
370
371
372 def _check_galaxy(ctx, **kwds):
373 """ Find Galaxy root, return None to indicate it should be
374 installed automatically.
375 """
376 install_galaxy = kwds.get("install_galaxy", None)
377 galaxy_root = None
378 if not install_galaxy:
379 galaxy_root = _find_galaxy_root(ctx, **kwds)
380 return galaxy_root
381
382
383 def _find_galaxy_root(ctx, **kwds):
384 galaxy_root = kwds.get("galaxy_root", None)
385 if galaxy_root:
386 return galaxy_root
387 elif ctx.global_config.get("galaxy_root", None):
388 return ctx.global_config["galaxy_root"]
389 else:
390 par_dir = os.getcwd()
391 while True:
392 run = os.path.join(par_dir, "run.sh")
393 config = os.path.join(par_dir, "config")
394 if os.path.isfile(run) and os.path.isdir(config):
395 return par_dir
396 new_par_dir = os.path.dirname(par_dir)
397 if new_par_dir == par_dir:
398 break
399 par_dir = new_par_dir
400 return None
401
402
403 def _find_test_data(tool_paths, **kwds):
404 path = "."
405 if len(tool_paths) > 0:
406 path = tool_paths[0]
407
408 # Find test data directory associated with path.
409 test_data = kwds.get("test_data", None)
410 if test_data:
411 return os.path.abspath(test_data)
412 else:
413 test_data = _search_tool_path_for(path, "test-data")
414 if test_data:
415 return test_data
416 warn(NO_TEST_DATA_MESSAGE)
417 return None
418
419
420 def _find_tool_data_table(tool_paths, test_data_dir, **kwds):
421 path = "."
422 if len(tool_paths) > 0:
423 path = tool_paths[0]
424
425 tool_data_table = kwds.get("tool_data_table", None)
426 if tool_data_table:
427 return os.path.abspath(tool_data_table)
428 else:
429 extra_paths = [test_data_dir] if test_data_dir else []
430 return _search_tool_path_for(
431 path,
432 "tool_data_table_conf.xml.test",
433 extra_paths,
434 ) or _search_tool_path_for( # if all else fails just use sample
435 path,
436 "tool_data_table_conf.xml.sample"
437 )
438
439
440 def _search_tool_path_for(path, target, extra_paths=[]):
441 if not os.path.isdir(path):
442 tool_dir = os.path.dirname(path)
443 else:
444 tool_dir = path
445 possible_dirs = [tool_dir, "."] + extra_paths
446 for possible_dir in possible_dirs:
447 possible_path = os.path.join(possible_dir, target)
448 if os.path.exists(possible_path):
449 return os.path.abspath(possible_path)
450 return None
451
452
453 def _configure_sheds_config_file(ctx, config_directory, **kwds):
454 if "shed_target" not in kwds:
455 kwds = kwds.copy()
456 kwds["shed_target"] = "toolshed"
457 shed_target_url = tool_shed_url(ctx, **kwds)
458 contents = _sub(TOOL_SHEDS_CONF, {"shed_target_url": shed_target_url})
459 tool_sheds_conf = os.path.join(config_directory, "tool_sheds_conf.xml")
460 write_file(tool_sheds_conf, contents)
461 return tool_sheds_conf
462
463
464 def _tool_conf_entry_for(tool_paths):
465 tool_definitions = ""
466 for tool_path in tool_paths:
467 if os.path.isdir(tool_path):
468 tool_definitions += '''<tool_dir dir="%s" />''' % tool_path
469 else:
470 tool_definitions += '''<tool file="%s" />''' % tool_path
471 return tool_definitions
472
473
474 def _shed_tool_conf(install_galaxy, config_directory):
475 # TODO: There is probably a reason this is split up like this but I have
476 # no clue why I did it and not documented on the commit message.
477 if install_galaxy:
478 config_dir = os.path.join(config_directory, "galaxy-dev", "config")
479 else:
480 config_dir = config_directory
481 return os.path.join(config_dir, "shed_tool_conf.xml")
482
483
484 def _install_galaxy(ctx, config_directory, kwds):
485 if not kwds.get("no_cache_galaxy", False):
486 _install_galaxy_via_git(ctx, config_directory, kwds)
487 else:
488 _install_galaxy_via_download(config_directory, kwds)
489
490
491 def _install_galaxy_via_download(config_directory, kwds):
492 command = galaxy_run.DOWNLOAD_GALAXY + "; tar -zxvf dev | tail"
493 _install_with_command(config_directory, command)
494
495
496 def _install_galaxy_via_git(ctx, config_directory, kwds):
497 _ensure_galaxy_repository_available(ctx)
498 workspace = ctx.workspace
499 gx_repo = os.path.join(workspace, "gx_repo")
500 command = git.command_clone(ctx, gx_repo, "galaxy-dev")
501 _install_with_command(config_directory, command)
502
503
504 def _build_eggs_cache(ctx, env, kwds):
505 if kwds.get("no_cache_galaxy", False):
506 return None
507 workspace = ctx.workspace
508 eggs_path = os.path.join(workspace, "gx_eggs")
509 if not os.path.exists(eggs_path):
510 os.makedirs(eggs_path)
511 env["GALAXY_EGGS_PATH"] = eggs_path
512
513
514 def _install_with_command(config_directory, command):
515 install_cmds = [
516 "cd %s" % config_directory,
517 command,
518 "cd galaxy-dev",
519 "type virtualenv >/dev/null 2>&1 && virtualenv .venv",
520 galaxy_run.ACTIVATE_COMMAND,
521 ]
522 shell(";".join(install_cmds))
523
524
525 def _ensure_galaxy_repository_available(ctx):
526 workspace = ctx.workspace
527 gx_repo = os.path.join(workspace, "gx_repo")
528 if os.path.exists(gx_repo):
529 # Attempt fetch - but don't fail if not interweb, etc...
530 shell("git --git-dir %s fetch >/dev/null 2>&1" % gx_repo)
531 else:
532 remote_repo = "https://github.com/galaxyproject/galaxy"
533 command = git.command_clone(ctx, remote_repo, gx_repo, bare=True)
534 shell(command)
535
536
537 def _build_env_for_galaxy(properties, template_args):
538 env = {}
539 for key, value in iteritems(properties):
540 var = "GALAXY_CONFIG_OVERRIDE_%s" % key.upper()
541 value = _sub(value, template_args)
542 env[var] = value
543 return env
544
545
546 def _build_test_env(properties, env):
547 # Keeping these environment variables around for a little while but they
548 # many are probably not needed as of the following commit.
549 # https://bitbucket.org/galaxy/galaxy-central/commits/d7dd1f9
550 test_property_variants = {
551 'GALAXY_TEST_MIGRATED_TOOL_CONF': 'migrated_tools_config',
552 'GALAXY_TEST_TOOL_CONF': 'tool_config_file',
553 'GALAXY_TEST_FILE_DIR': 'test_data_dir',
554 'GALAXY_TOOL_DEPENDENCY_DIR': 'tool_dependency_dir',
555 # Next line would be required for tool shed tests.
556 # 'GALAXY_TEST_TOOL_DEPENDENCY_DIR': 'tool_dependency_dir',
557 }
558 for test_key, gx_key in test_property_variants.items():
559 value = properties.get(gx_key, None)
560 if value is not None:
561 env[test_key] = value
562
563
564 def _handle_dependency_resolution(config_directory, kwds):
565 resolutions_strategies = [
566 "brew_dependency_resolution",
567 "dependency_resolvers_config_file",
568 "shed_dependency_resolution",
569 ]
570
571 selected_strategies = 0
572 for key in resolutions_strategies:
573 if kwds.get(key):
574 selected_strategies += 1
575
576 if selected_strategies > 1:
577 message = "At most one option from [%s] may be specified"
578 raise click.UsageError(message % resolutions_strategies)
579
580 for key in STOCK_DEPENDENCY_RESOLUTION_STRATEGIES:
581 if kwds.get(key):
582 resolvers_conf = os.path.join(
583 config_directory,
584 "resolvers_conf.xml"
585 )
586 conf_contents = STOCK_DEPENDENCY_RESOLUTION_STRATEGIES[key]
587 open(resolvers_conf, "w").write(conf_contents)
588 kwds["dependency_resolvers_config_file"] = resolvers_conf
589
590
591 def _handle_job_metrics(config_directory, kwds):
592 metrics_conf = os.path.join(config_directory, "job_metrics_conf.xml")
593 open(metrics_conf, "w").write(EMPTY_JOB_METRICS_TEMPLATE)
594 kwds["job_metrics_config_file"] = metrics_conf
595
596
597 def _handle_kwd_overrides(properties, kwds):
598 kwds_gx_properties = [
599 'job_config_file',
600 'job_metrics_config_file',
601 'dependency_resolvers_config_file',
602 'tool_dependency_dir',
603 ]
604 for prop in kwds_gx_properties:
605 val = kwds.get(prop, None)
606 if val:
607 properties[prop] = val
608
609
610 def _sub(template, args):
611 if template is None:
612 return ''
613 return Template(template).safe_substitute(args)
614
615 __all__ = ["galaxy_config"]