Mercurial > repos > bcclaywell > argo_navis

comparison venv/lib/python2.7/site-packages/jinja2/ext.py @ 0:d67268158946 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
planemo upload commit a3f181f5f126803c654b3a66dd4e83a48f7e203b
author bcclaywell
date Mon, 12 Oct 2015 17:43:33 -0400
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:d67268158946
1 # -*- coding: utf-8 -*-
2 """
3 jinja2.ext
4 ~~~~~~~~~~
5
6 Jinja extensions allow to add custom tags similar to the way django custom
7 tags work. By default two example extensions exist: an i18n and a cache
8 extension.
9
10 :copyright: (c) 2010 by the Jinja Team.
11 :license: BSD.
12 """
13 from jinja2 import nodes
14 from jinja2.defaults import BLOCK_START_STRING, \
15 BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
16 COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \
17 LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \
18 KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS
19 from jinja2.environment import Environment
20 from jinja2.runtime import concat
21 from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
22 from jinja2.utils import contextfunction, import_string, Markup
23 from jinja2._compat import with_metaclass, string_types, iteritems
24
25
26 # the only real useful gettext functions for a Jinja template. Note
27 # that ugettext must be assigned to gettext as Jinja doesn't support
28 # non unicode strings.
29 GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
30
31
32 class ExtensionRegistry(type):
33 """Gives the extension an unique identifier."""
34
35 def __new__(cls, name, bases, d):
36 rv = type.__new__(cls, name, bases, d)
37 rv.identifier = rv.__module__ + '.' + rv.__name__
38 return rv
39
40
41 class Extension(with_metaclass(ExtensionRegistry, object)):
42 """Extensions can be used to add extra functionality to the Jinja template
43 system at the parser level. Custom extensions are bound to an environment
44 but may not store environment specific data on `self`. The reason for
45 this is that an extension can be bound to another environment (for
46 overlays) by creating a copy and reassigning the `environment` attribute.
47
48 As extensions are created by the environment they cannot accept any
49 arguments for configuration. One may want to work around that by using
50 a factory function, but that is not possible as extensions are identified
51 by their import name. The correct way to configure the extension is
52 storing the configuration values on the environment. Because this way the
53 environment ends up acting as central configuration storage the
54 attributes may clash which is why extensions have to ensure that the names
55 they choose for configuration are not too generic. ``prefix`` for example
56 is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
57 name as includes the name of the extension (fragment cache).
58 """
59
60 #: if this extension parses this is the list of tags it's listening to.
61 tags = set()
62
63 #: the priority of that extension. This is especially useful for
64 #: extensions that preprocess values. A lower value means higher
65 #: priority.
66 #:
67 #: .. versionadded:: 2.4
68 priority = 100
69
70 def __init__(self, environment):
71 self.environment = environment
72
73 def bind(self, environment):
74 """Create a copy of this extension bound to another environment."""
75 rv = object.__new__(self.__class__)
76 rv.__dict__.update(self.__dict__)
77 rv.environment = environment
78 return rv
79
80 def preprocess(self, source, name, filename=None):
81 """This method is called before the actual lexing and can be used to
82 preprocess the source. The `filename` is optional. The return value
83 must be the preprocessed source.
84 """
85 return source
86
87 def filter_stream(self, stream):
88 """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
89 to filter tokens returned. This method has to return an iterable of
90 :class:`~jinja2.lexer.Token`\s, but it doesn't have to return a
91 :class:`~jinja2.lexer.TokenStream`.
92
93 In the `ext` folder of the Jinja2 source distribution there is a file
94 called `inlinegettext.py` which implements a filter that utilizes this
95 method.
96 """
97 return stream
98
99 def parse(self, parser):
100 """If any of the :attr:`tags` matched this method is called with the
101 parser as first argument. The token the parser stream is pointing at
102 is the name token that matched. This method has to return one or a
103 list of multiple nodes.
104 """
105 raise NotImplementedError()
106
107 def attr(self, name, lineno=None):
108 """Return an attribute node for the current extension. This is useful
109 to pass constants on extensions to generated template code.
110
111 ::
112
113 self.attr('_my_attribute', lineno=lineno)
114 """
115 return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
116
117 def call_method(self, name, args=None, kwargs=None, dyn_args=None,
118 dyn_kwargs=None, lineno=None):
119 """Call a method of the extension. This is a shortcut for
120 :meth:`attr` + :class:`jinja2.nodes.Call`.
121 """
122 if args is None:
123 args = []
124 if kwargs is None:
125 kwargs = []
126 return nodes.Call(self.attr(name, lineno=lineno), args, kwargs,
127 dyn_args, dyn_kwargs, lineno=lineno)
128
129
130 @contextfunction
131 def _gettext_alias(__context, *args, **kwargs):
132 return __context.call(__context.resolve('gettext'), *args, **kwargs)
133
134
135 def _make_new_gettext(func):
136 @contextfunction
137 def gettext(__context, __string, **variables):
138 rv = __context.call(func, __string)
139 if __context.eval_ctx.autoescape:
140 rv = Markup(rv)
141 return rv % variables
142 return gettext
143
144
145 def _make_new_ngettext(func):
146 @contextfunction
147 def ngettext(__context, __singular, __plural, __num, **variables):
148 variables.setdefault('num', __num)
149 rv = __context.call(func, __singular, __plural, __num)
150 if __context.eval_ctx.autoescape:
151 rv = Markup(rv)
152 return rv % variables
153 return ngettext
154
155
156 class InternationalizationExtension(Extension):
157 """This extension adds gettext support to Jinja2."""
158 tags = set(['trans'])
159
160 # TODO: the i18n extension is currently reevaluating values in a few
161 # situations. Take this example:
162 # {% trans count=something() %}{{ count }} foo{% pluralize
163 # %}{{ count }} fooss{% endtrans %}
164 # something is called twice here. One time for the gettext value and
165 # the other time for the n-parameter of the ngettext function.
166
167 def __init__(self, environment):
168 Extension.__init__(self, environment)
169 environment.globals['_'] = _gettext_alias
170 environment.extend(
171 install_gettext_translations=self._install,
172 install_null_translations=self._install_null,
173 install_gettext_callables=self._install_callables,
174 uninstall_gettext_translations=self._uninstall,
175 extract_translations=self._extract,
176 newstyle_gettext=False
177 )
178
179 def _install(self, translations, newstyle=None):
180 gettext = getattr(translations, 'ugettext', None)
181 if gettext is None:
182 gettext = translations.gettext
183 ngettext = getattr(translations, 'ungettext', None)
184 if ngettext is None:
185 ngettext = translations.ngettext
186 self._install_callables(gettext, ngettext, newstyle)
187
188 def _install_null(self, newstyle=None):
189 self._install_callables(
190 lambda x: x,
191 lambda s, p, n: (n != 1 and (p,) or (s,))[0],
192 newstyle
193 )
194
195 def _install_callables(self, gettext, ngettext, newstyle=None):
196 if newstyle is not None:
197 self.environment.newstyle_gettext = newstyle
198 if self.environment.newstyle_gettext:
199 gettext = _make_new_gettext(gettext)
200 ngettext = _make_new_ngettext(ngettext)
201 self.environment.globals.update(
202 gettext=gettext,
203 ngettext=ngettext
204 )
205
206 def _uninstall(self, translations):
207 for key in 'gettext', 'ngettext':
208 self.environment.globals.pop(key, None)
209
210 def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
211 if isinstance(source, string_types):
212 source = self.environment.parse(source)
213 return extract_from_ast(source, gettext_functions)
214
215 def parse(self, parser):
216 """Parse a translatable tag."""
217 lineno = next(parser.stream).lineno
218 num_called_num = False
219
220 # find all the variables referenced. Additionally a variable can be
221 # defined in the body of the trans block too, but this is checked at
222 # a later state.
223 plural_expr = None
224 plural_expr_assignment = None
225 variables = {}
226 while parser.stream.current.type != 'block_end':
227 if variables:
228 parser.stream.expect('comma')
229
230 # skip colon for python compatibility
231 if parser.stream.skip_if('colon'):
232 break
233
234 name = parser.stream.expect('name')
235 if name.value in variables:
236 parser.fail('translatable variable %r defined twice.' %
237 name.value, name.lineno,
238 exc=TemplateAssertionError)
239
240 # expressions
241 if parser.stream.current.type == 'assign':
242 next(parser.stream)
243 variables[name.value] = var = parser.parse_expression()
244 else:
245 variables[name.value] = var = nodes.Name(name.value, 'load')
246
247 if plural_expr is None:
248 if isinstance(var, nodes.Call):
249 plural_expr = nodes.Name('_trans', 'load')
250 variables[name.value] = plural_expr
251 plural_expr_assignment = nodes.Assign(
252 nodes.Name('_trans', 'store'), var)
253 else:
254 plural_expr = var
255 num_called_num = name.value == 'num'
256
257 parser.stream.expect('block_end')
258
259 plural = plural_names = None
260 have_plural = False
261 referenced = set()
262
263 # now parse until endtrans or pluralize
264 singular_names, singular = self._parse_block(parser, True)
265 if singular_names:
266 referenced.update(singular_names)
267 if plural_expr is None:
268 plural_expr = nodes.Name(singular_names[0], 'load')
269 num_called_num = singular_names[0] == 'num'
270
271 # if we have a pluralize block, we parse that too
272 if parser.stream.current.test('name:pluralize'):
273 have_plural = True
274 next(parser.stream)
275 if parser.stream.current.type != 'block_end':
276 name = parser.stream.expect('name')
277 if name.value not in variables:
278 parser.fail('unknown variable %r for pluralization' %
279 name.value, name.lineno,
280 exc=TemplateAssertionError)
281 plural_expr = variables[name.value]
282 num_called_num = name.value == 'num'
283 parser.stream.expect('block_end')
284 plural_names, plural = self._parse_block(parser, False)
285 next(parser.stream)
286 referenced.update(plural_names)
287 else:
288 next(parser.stream)
289
290 # register free names as simple name expressions
291 for var in referenced:
292 if var not in variables:
293 variables[var] = nodes.Name(var, 'load')
294
295 if not have_plural:
296 plural_expr = None
297 elif plural_expr is None:
298 parser.fail('pluralize without variables', lineno)
299
300 node = self._make_node(singular, plural, variables, plural_expr,
301 bool(referenced),
302 num_called_num and have_plural)
303 node.set_lineno(lineno)
304 if plural_expr_assignment is not None:
305 return [plural_expr_assignment, node]
306 else:
307 return node
308
309 def _parse_block(self, parser, allow_pluralize):
310 """Parse until the next block tag with a given name."""
311 referenced = []
312 buf = []
313 while 1:
314 if parser.stream.current.type == 'data':
315 buf.append(parser.stream.current.value.replace('%', '%%'))
316 next(parser.stream)
317 elif parser.stream.current.type == 'variable_begin':
318 next(parser.stream)
319 name = parser.stream.expect('name').value
320 referenced.append(name)
321 buf.append('%%(%s)s' % name)
322 parser.stream.expect('variable_end')
323 elif parser.stream.current.type == 'block_begin':
324 next(parser.stream)
325 if parser.stream.current.test('name:endtrans'):
326 break
327 elif parser.stream.current.test('name:pluralize'):
328 if allow_pluralize:
329 break
330 parser.fail('a translatable section can have only one '
331 'pluralize section')
332 parser.fail('control structures in translatable sections are '
333 'not allowed')
334 elif parser.stream.eos:
335 parser.fail('unclosed translation block')
336 else:
337 assert False, 'internal parser error'
338
339 return referenced, concat(buf)
340
341 def _make_node(self, singular, plural, variables, plural_expr,
342 vars_referenced, num_called_num):
343 """Generates a useful node from the data provided."""
344 # no variables referenced? no need to escape for old style
345 # gettext invocations only if there are vars.
346 if not vars_referenced and not self.environment.newstyle_gettext:
347 singular = singular.replace('%%', '%')
348 if plural:
349 plural = plural.replace('%%', '%')
350
351 # singular only:
352 if plural_expr is None:
353 gettext = nodes.Name('gettext', 'load')
354 node = nodes.Call(gettext, [nodes.Const(singular)],
355 [], None, None)
356
357 # singular and plural
358 else:
359 ngettext = nodes.Name('ngettext', 'load')
360 node = nodes.Call(ngettext, [
361 nodes.Const(singular),
362 nodes.Const(plural),
363 plural_expr
364 ], [], None, None)
365
366 # in case newstyle gettext is used, the method is powerful
367 # enough to handle the variable expansion and autoescape
368 # handling itself
369 if self.environment.newstyle_gettext:
370 for key, value in iteritems(variables):
371 # the function adds that later anyways in case num was
372 # called num, so just skip it.
373 if num_called_num and key == 'num':
374 continue
375 node.kwargs.append(nodes.Keyword(key, value))
376
377 # otherwise do that here
378 else:
379 # mark the return value as safe if we are in an
380 # environment with autoescaping turned on
381 node = nodes.MarkSafeIfAutoescape(node)
382 if variables:
383 node = nodes.Mod(node, nodes.Dict([
384 nodes.Pair(nodes.Const(key), value)
385 for key, value in variables.items()
386 ]))
387 return nodes.Output([node])
388
389
390 class ExprStmtExtension(Extension):
391 """Adds a `do` tag to Jinja2 that works like the print statement just
392 that it doesn't print the return value.
393 """
394 tags = set(['do'])
395
396 def parse(self, parser):
397 node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
398 node.node = parser.parse_tuple()
399 return node
400
401
402 class LoopControlExtension(Extension):
403 """Adds break and continue to the template engine."""
404 tags = set(['break', 'continue'])
405
406 def parse(self, parser):
407 token = next(parser.stream)
408 if token.value == 'break':
409 return nodes.Break(lineno=token.lineno)
410 return nodes.Continue(lineno=token.lineno)
411
412
413 class WithExtension(Extension):
414 """Adds support for a django-like with block."""
415 tags = set(['with'])
416
417 def parse(self, parser):
418 node = nodes.Scope(lineno=next(parser.stream).lineno)
419 assignments = []
420 while parser.stream.current.type != 'block_end':
421 lineno = parser.stream.current.lineno
422 if assignments:
423 parser.stream.expect('comma')
424 target = parser.parse_assign_target()
425 parser.stream.expect('assign')
426 expr = parser.parse_expression()
427 assignments.append(nodes.Assign(target, expr, lineno=lineno))
428 node.body = assignments + \
429 list(parser.parse_statements(('name:endwith',),
430 drop_needle=True))
431 return node
432
433
434 class AutoEscapeExtension(Extension):
435 """Changes auto escape rules for a scope."""
436 tags = set(['autoescape'])
437
438 def parse(self, parser):
439 node = nodes.ScopedEvalContextModifier(lineno=next(parser.stream).lineno)
440 node.options = [
441 nodes.Keyword('autoescape', parser.parse_expression())
442 ]
443 node.body = parser.parse_statements(('name:endautoescape',),
444 drop_needle=True)
445 return nodes.Scope([node])
446
447
448 def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
449 babel_style=True):
450 """Extract localizable strings from the given template node. Per
451 default this function returns matches in babel style that means non string
452 parameters as well as keyword arguments are returned as `None`. This
453 allows Babel to figure out what you really meant if you are using
454 gettext functions that allow keyword arguments for placeholder expansion.
455 If you don't want that behavior set the `babel_style` parameter to `False`
456 which causes only strings to be returned and parameters are always stored
457 in tuples. As a consequence invalid gettext calls (calls without a single
458 string parameter or string parameters after non-string parameters) are
459 skipped.
460
461 This example explains the behavior:
462
463 >>> from jinja2 import Environment
464 >>> env = Environment()
465 >>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
466 >>> list(extract_from_ast(node))
467 [(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
468 >>> list(extract_from_ast(node, babel_style=False))
469 [(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
470
471 For every string found this function yields a ``(lineno, function,
472 message)`` tuple, where:
473
474 * ``lineno`` is the number of the line on which the string was found,
475 * ``function`` is the name of the ``gettext`` function used (if the
476 string was extracted from embedded Python code), and
477 * ``message`` is the string itself (a ``unicode`` object, or a tuple
478 of ``unicode`` objects for functions with multiple string arguments).
479
480 This extraction function operates on the AST and is because of that unable
481 to extract any comments. For comment support you have to use the babel
482 extraction interface or extract comments yourself.
483 """
484 for node in node.find_all(nodes.Call):
485 if not isinstance(node.node, nodes.Name) or \
486 node.node.name not in gettext_functions:
487 continue
488
489 strings = []
490 for arg in node.args:
491 if isinstance(arg, nodes.Const) and \
492 isinstance(arg.value, string_types):
493 strings.append(arg.value)
494 else:
495 strings.append(None)
496
497 for arg in node.kwargs:
498 strings.append(None)
499 if node.dyn_args is not None:
500 strings.append(None)
501 if node.dyn_kwargs is not None:
502 strings.append(None)
503
504 if not babel_style:
505 strings = tuple(x for x in strings if x is not None)
506 if not strings:
507 continue
508 else:
509 if len(strings) == 1:
510 strings = strings[0]
511 else:
512 strings = tuple(strings)
513 yield node.lineno, node.node.name, strings
514
515
516 class _CommentFinder(object):
517 """Helper class to find comments in a token stream. Can only
518 find comments for gettext calls forwards. Once the comment
519 from line 4 is found, a comment for line 1 will not return a
520 usable value.
521 """
522
523 def __init__(self, tokens, comment_tags):
524 self.tokens = tokens
525 self.comment_tags = comment_tags
526 self.offset = 0
527 self.last_lineno = 0
528
529 def find_backwards(self, offset):
530 try:
531 for _, token_type, token_value in \
532 reversed(self.tokens[self.offset:offset]):
533 if token_type in ('comment', 'linecomment'):
534 try:
535 prefix, comment = token_value.split(None, 1)
536 except ValueError:
537 continue
538 if prefix in self.comment_tags:
539 return [comment.rstrip()]
540 return []
541 finally:
542 self.offset = offset
543
544 def find_comments(self, lineno):
545 if not self.comment_tags or self.last_lineno > lineno:
546 return []
547 for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
548 if token_lineno > lineno:
549 return self.find_backwards(self.offset + idx)
550 return self.find_backwards(len(self.tokens))
551
552
553 def babel_extract(fileobj, keywords, comment_tags, options):
554 """Babel extraction method for Jinja templates.
555
556 .. versionchanged:: 2.3
557 Basic support for translation comments was added. If `comment_tags`
558 is now set to a list of keywords for extraction, the extractor will
559 try to find the best preceeding comment that begins with one of the
560 keywords. For best results, make sure to not have more than one
561 gettext call in one line of code and the matching comment in the
562 same line or the line before.
563
564 .. versionchanged:: 2.5.1
565 The `newstyle_gettext` flag can be set to `True` to enable newstyle
566 gettext calls.
567
568 .. versionchanged:: 2.7
569 A `silent` option can now be provided. If set to `False` template
570 syntax errors are propagated instead of being ignored.
571
572 :param fileobj: the file-like object the messages should be extracted from
573 :param keywords: a list of keywords (i.e. function names) that should be
574 recognized as translation functions
575 :param comment_tags: a list of translator tags to search for and include
576 in the results.
577 :param options: a dictionary of additional options (optional)
578 :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
579 (comments will be empty currently)
580 """
581 extensions = set()
582 for extension in options.get('extensions', '').split(','):
583 extension = extension.strip()
584 if not extension:
585 continue
586 extensions.add(import_string(extension))
587 if InternationalizationExtension not in extensions:
588 extensions.add(InternationalizationExtension)
589
590 def getbool(options, key, default=False):
591 return options.get(key, str(default)).lower() in \
592 ('1', 'on', 'yes', 'true')
593
594 silent = getbool(options, 'silent', True)
595 environment = Environment(
596 options.get('block_start_string', BLOCK_START_STRING),
597 options.get('block_end_string', BLOCK_END_STRING),
598 options.get('variable_start_string', VARIABLE_START_STRING),
599 options.get('variable_end_string', VARIABLE_END_STRING),
600 options.get('comment_start_string', COMMENT_START_STRING),
601 options.get('comment_end_string', COMMENT_END_STRING),
602 options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
603 options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
604 getbool(options, 'trim_blocks', TRIM_BLOCKS),
605 getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS),
606 NEWLINE_SEQUENCE,
607 getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE),
608 frozenset(extensions),
609 cache_size=0,
610 auto_reload=False
611 )
612
613 if getbool(options, 'newstyle_gettext'):
614 environment.newstyle_gettext = True
615
616 source = fileobj.read().decode(options.get('encoding', 'utf-8'))
617 try:
618 node = environment.parse(source)
619 tokens = list(environment.lex(environment.preprocess(source)))
620 except TemplateSyntaxError as e:
621 if not silent:
622 raise
623 # skip templates with syntax errors
624 return
625
626 finder = _CommentFinder(tokens, comment_tags)
627 for lineno, func, message in extract_from_ast(node, keywords):
628 yield lineno, func, message, finder.find_comments(lineno)
629
630
631 #: nicer import names
632 i18n = InternationalizationExtension
633 do = ExprStmtExtension
634 loopcontrols = LoopControlExtension
635 with_ = WithExtension
636 autoescape = AutoEscapeExtension