# # The MIT License (MIT) # # Copyright (c) 2015 Philippe Faist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. # from __future__ import print_function #, absolute_import import re from collections import namedtuple import sys import logging if sys.version_info.major > 2: def unicode(string): return string basestring = str logger = logging.getLogger(__name__) class LatexWalkerError(Exception): pass class LatexWalkerParseError(LatexWalkerError): def __init__(self, msg, s=None, pos=None): self.msg = msg self.s = s self.pos = pos disp = '...'+s[max(pos-25,0):pos] disp = '\n%s\n'%(disp) + (' '*len(disp)) + s[pos:pos+25]+'...' LatexWalkerError.__init__(self, msg + ( " @ %d:\n%s" %(pos, disp) )) class LatexWalkerEndOfStream(LatexWalkerError): pass MacrosDef = namedtuple('MacrosDef', ['macname', 'optarg', 'numargs']) """ Class which stores a Macro syntax. - `macname` stores the name of the macro, without the leading backslash. - `optarg` may be one of `True`, `False`, or `None`. + if `True`, the macro expects as first argument an optional argument in square brackets. Then, `numargs` specifies the number of additional mandatory arguments to the command, given in usual curly braces (or simply as one TeX token) + if `False`, the macro only expects a number of mandatory arguments given by `numargs`. The mandatory arguments are given in usual curly braces (or simply as one TeX token) + if `None`, then `numargs` is a string of either characters "{" or "[", in which each curly brace specifies a mandatory argument and each square bracket specifies an optional argument in square brackets. For example, "{{[{" expects two mandatory arguments, then an optional argument in square brackets, and then another mandatory argument. """ _default_macro_list = ( MacrosDef('documentclass', True, 1), MacrosDef('usepackage', True, 1), MacrosDef('selectlanguage', True, 1), MacrosDef('setlength', True, 2), MacrosDef('addlength', True, 2), MacrosDef('setcounter', True, 2), MacrosDef('addcounter', True, 2), MacrosDef('newcommand', None, "{[{"), MacrosDef('renewcommand', None, "{[{"), MacrosDef('DeclareMathOperator', False, 2), MacrosDef('input', False, 1), MacrosDef('hspace', False, 1), MacrosDef('vspace', False, 1), MacrosDef('\\', True, 0), # (Note: single backslash) end of line with optional spacing, e.g. \\[2mm] MacrosDef('item', True, 0), # \input{someotherfile} MacrosDef('input', False, 1), MacrosDef('include', False, 1), MacrosDef('includegraphics', True, 1), MacrosDef('textit', False, 1), MacrosDef('textbf', False, 1), MacrosDef('textsc', False, 1), MacrosDef('textsl', False, 1), MacrosDef('text', False, 1), MacrosDef('mathrm', False, 1), MacrosDef('label', False, 1), MacrosDef('ref', False, 1), MacrosDef('eqref', False, 1), MacrosDef('url', False, 1), MacrosDef('hypersetup', False, 1), MacrosDef('footnote', True, 1), MacrosDef('keywords', False, 1), MacrosDef('hphantom', True, 1), MacrosDef('vphantom', True, 1), MacrosDef("'", False, 1), MacrosDef("`", False, 1), MacrosDef('"', False, 1), MacrosDef("c", False, 1), MacrosDef("^", False, 1), MacrosDef("~", False, 1), MacrosDef("H", False, 1), MacrosDef("k", False, 1), MacrosDef("=", False, 1), MacrosDef("b", False, 1), MacrosDef(".", False, 1), MacrosDef("d", False, 1), MacrosDef("r", False, 1), MacrosDef("u", False, 1), MacrosDef("v", False, 1), MacrosDef("vec", False, 1), MacrosDef("dot", False, 1), MacrosDef("hat", False, 1), MacrosDef("check", False, 1), MacrosDef("breve", False, 1), MacrosDef("acute", False, 1), MacrosDef("grave", False, 1), MacrosDef("tilde", False, 1), MacrosDef("bar", False, 1), MacrosDef("ddot", False, 1), MacrosDef('frac', False, 2), MacrosDef('nicefrac', False, 2), MacrosDef('sqrt', True, 1), MacrosDef('ket', False, 1), MacrosDef('bra', False, 1), MacrosDef('braket', False, 2), MacrosDef('ketbra', False, 2), MacrosDef('texorpdfstring', False, 2), # ethuebung MacrosDef('UebungLoesungFont', False, 1), MacrosDef('UebungHinweisFont', False, 1), MacrosDef('UebungExTitleFont', False, 1), MacrosDef('UebungSubExTitleFont', False, 1), MacrosDef('UebungTipsFont', False, 1), MacrosDef('UebungLabel', False, 1), MacrosDef('UebungSubLabel', False, 1), MacrosDef('UebungLabelEnum', False, 1), MacrosDef('UebungLabelEnumSub', False, 1), MacrosDef('UebungSolLabel', False, 1), MacrosDef('UebungHinweisLabel', False, 1), MacrosDef('UebungHinweiseLabel', False, 1), MacrosDef('UebungSolEquationLabel', False, 1), MacrosDef('UebungTipsLabel', False, 1), MacrosDef('UebungTipsEquationLabel', False, 1), MacrosDef('UebungsblattTitleSeries', False, 1), MacrosDef('UebungsblattTitleSolutions', False, 1), MacrosDef('UebungsblattTitleTips', False, 1), MacrosDef('UebungsblattNumber', False, 1), MacrosDef('UebungsblattTitleFont', False, 1), MacrosDef('UebungTitleCenterVSpacing', False, 1), MacrosDef('UebungAttachedSolutionTitleTop', False, 1), MacrosDef('UebungAttachedSolutionTitleFont', False, 1), MacrosDef('UebungAttachedSolutionTitle', False, 1), MacrosDef('UebungTextAttachedSolution', False, 1), MacrosDef('UebungDueByLabel', False, 1), MacrosDef('UebungDueBy', False, 1), MacrosDef('UebungLecture', False, 1), MacrosDef('UebungProf', False, 1), MacrosDef('UebungLecturer', False, 1), MacrosDef('UebungSemester', False, 1), MacrosDef('UebungLogoFile', False, 1), MacrosDef('UebungLanguage', False, 1), MacrosDef('UebungStyle', False, 1), # MacrosDef('uebung', False, '{['), MacrosDef('exercise', False, '{['), MacrosDef('keywords', False, 1), MacrosDef('subuebung', False, 1), MacrosDef('subexercise', False, 1), MacrosDef('pdfloesung', True, 1), MacrosDef('pdfsolution', True, 1), MacrosDef('exenumfulllabel', False, 1), MacrosDef('hint', False, 1), MacrosDef('hints', False, 1), MacrosDef('hinweis', False, 1), MacrosDef('hinweise', False, 1), ) default_macro_dict = dict([(m.macname, m) for m in _default_macro_list]) # ------------------------------------------------ class LatexToken(object): r""" Represents a token read from the LaTeX input. This is not quite a LaTeX token, it's just a part of the input which we treat in the same way (e.g. a bunch of content characters, a comment, a macro, etc.) Information about the object is stored into the fields `tok` and `arg`. The `tok` field is a string which identifies the type of the token. The `arg` depends on what `tok` is, and describes the actual input. Additionally, this class stores information about the position of the token in the input stream in the field `pos`. This `pos` is an integer which corresponds to the index in the input string. The field `len` stores the length of the token in the input string. This means that this token spans in the input string from `pos` to `pos+len`. Leading whitespace before the token is not returned as a separate 'char'-type token, but it is given in the `pre_space` field of the token which follows. Pre-space may contain a newline, but not two consecutive newlines. The `tok` field may be one of: - 'char': raw characters which have no special LaTeX meaning; they are part of the text content. The `arg` field contains the characters themselves. - 'macro': a macro invokation, but not '\begin' or '\end' The `arg` field contains the name of the macro, without the leading backslash. - 'begin_environment': an invokation of '\begin{environment}'. The `arg` field contains the name of the environment inside the braces. - 'end_environment': an invokation of '\end{environment}'. The `arg` field contains the name of the environment inside the braces. - 'comment': a LaTeX comment delimited by a percent sign up to the end of the line. The `arg` field contains the text in the comment line, not including the percent sign nor the newline. - 'brace_open': an opening brace. This is usually a curly brace, and sometimes also a square bracket. What is parsed as a brace depends on the arguments to :py:func:`get_token()`. The `arg` is a string which contains the relevant brace character. - 'brace_close': a closing brace. This is usually a curly brace, and sometimes also a square bracket. What is parsed as a brace depends on the arguments to :py:func:`get_token()`. The `arg` is a string which contains the relevant brace character. - 'mathmode_inline': a delimiter which starts inline math. This is (e.g.) a single '$' character which is not part of a double '$$' display environment delimiter. The `arg` is the string value of the delimiter in question ('$') """ def __init__(self, tok, arg, pos, len, pre_space): self.tok = tok self.arg = arg self.pos = pos self.len = len self.pre_space = pre_space self._fields = ['tok', 'arg', 'pos', 'len', 'pre_space'] super(LatexToken, self).__init__() def __unicode__(self): return self.__repr__() def __repr__(self): return ( u"LatexToken(" + u", ".join([ u"%s=%r"%(k,getattr(self,k)) for k in self._fields ]) + u")" ) def __str__(self): return self.__unicode__().encode('utf-8') def __eq__(self, other): return all( ( getattr(self, f) == getattr(other, f) for f in self._fields ) ) # ------------------------------------------------ class LatexNode(object): """ Represents an abstract 'node' of the latex document. Use :py:meth:`nodeType()` to figure out what type of node this is. """ def __init__(self, **kwargs): """ Important: subclasses must set `self._fields` to a list (or tuple or iterable) of the fields stored in this object. """ super(LatexNode, self).__init__(**kwargs) def nodeType(self): """ Returns the class which corresponds to the type of this node. This is a Python class object, that is one of :py:class:`~pylatexenc.latexwalker.LatexCharsNode`, :py:class:`~pylatexenc.latexwalker.LatexGroupNode`, etc. """ return LatexNode def isNodeType(self, t): """ Returns `True` if the current node is of the given type. The argument `t` must be a Python class such as, e.g. :py:class:`~pylatexenc.latexwalker.LatexGroupNode`. """ return isinstance(self, t) def __eq__(self, other): return other is not None and self.nodeType() == other.nodeType() and all( ( getattr(self, f) == getattr(other, f) for f in self._fields ) ) def __unicode__(self): return self.__repr__() def __str__(self): return self.__unicode__().encode('utf-8') def __repr__(self): return ( self.nodeType().__name__ + u"(" + u", ".join([ u"%s=%r"%(k,getattr(self,k)) for k in self._fields ]) + u")" ) class LatexCharsNode(LatexNode): """ A string of characters in the LaTeX document, without any special meaning. """ def __init__(self, chars, **kwargs): r""" Arguments: - `chars`: the actual characters. """ super(LatexCharsNode, self).__init__(**kwargs) self._fields = ('chars',) self.chars = chars def nodeType(self): return LatexCharsNode class LatexGroupNode(LatexNode): r""" A LaTeX group, i.e. `{...}`. """ def __init__(self, nodelist, **kwargs): """ Arguments: - `nodelist`: a list of nodes which comprise the group. """ super(LatexNode, self).__init__(**kwargs) self._fields = ('nodelist',) self.nodelist = nodelist def nodeType(self): return LatexGroupNode class LatexCommentNode(LatexNode): """ A LaTeX comment, delimited by a percent sign until the end of line. """ def __init__(self, comment, **kwargs): super(LatexCommentNode, self).__init__(**kwargs) self._fields = ('comment',) self.comment = comment def nodeType(self): return LatexCommentNode class LatexMacroNode(LatexNode): r""" Represents a 'macro' type node, e.g. '\textbf' """ def __init__(self, macroname, nodeoptarg=None, nodeargs=[], **kwargs): r""" Represents a 'macro' type node, e.g. '\textbf' Arguments: - `macroname`: the name of the macro (string), *without* the leading backslash - `nodeoptarg`: if non-`None`, this corresponds to the optional argument of the macro - `nodeargs`: a list of arguments to the macro. Each item in the list should be a LatexNode. """ super(LatexMacroNode, self).__init__(**kwargs) self._fields = ('macroname','nodeoptarg','nodeargs',) self.macroname = macroname self.nodeoptarg = nodeoptarg self.nodeargs = nodeargs def nodeType(self): return LatexMacroNode class LatexEnvironmentNode(LatexNode): r""" A LaTeX Environment Node, i.e. `\begin{something} ... \end{something}`. See :py:meth:`__init__`. """ def __init__(self, envname, nodelist, optargs=[], args=[], **kwargs): r""" A LaTeX Environment Node, i.e. `\begin{something} ... \end{something}`. Arguments: - `envname`: the name of the environment ('itemize', 'equation', ...) - `nodelist`: a list of :py:class:`LatexNode`'s that represent all the contents between the `\begin{...}` instruction and the `\end{...}` instruction. - `optargs`: any possible optional argument passed to the `\begin{...}` instruction, for example in `\begin{enumerate}[label=\roman*)] (Currently, only a single optional argument is parsed, but this argument still accepts anyway a list of :py:class:`LatexNode`'s.) - `args`: any possible regular arguments passed to the `\begin{...}` instruction, for example in `\begin{tabular}{clr}`. Currently, only a single regular argument is parsed at maximum, but this is anyway a list of :py:class:`LatexNode`'s. """ super(LatexEnvironmentNode, self).__init__(**kwargs) self._fields = ('envname','nodelist','optargs','args',) self.envname = envname self.nodelist = nodelist self.optargs = optargs self.args = args def nodeType(self): return LatexEnvironmentNode class LatexMathNode(LatexNode): r""" A Math node type. See :py:meth:`__init__`. """ def __init__(self, displaytype, nodelist=[], **kwargs): r""" A Math node type. Note that currently only 'inline' math environments are detected. Arguments: - `displaytype`: either 'inline' or 'display' """ super(LatexMathNode, self).__init__(**kwargs) self._fields = ('displaytype','nodelist',) self.displaytype = displaytype self.nodelist = nodelist def nodeType(self): return LatexMathNode # ------------------------------------------------------------------------------ class _PushPropOverride(object): def __init__(self, obj, propname, new_value): super(_PushPropOverride, self).__init__() self.obj = obj self.propname = propname self.new_value = new_value def __enter__(self): if self.new_value is not None: self.initval = getattr(self.obj, self.propname) setattr(self.obj, self.propname, self.new_value) return self def __exit__(self, type, value, traceback): # clean-up if self.new_value is not None: setattr(self.obj, self.propname, self.initval) # ------------------------------------------------------------------------------ class LatexWalker(object): r""" A parser which walks through an input stream, parsing it as LaTeX markup. """ def __init__(self, s, macro_dict=None, **flags): self.s = s if macro_dict is not None: self.macro_dict = macro_dict else: self.macro_dict = default_macro_dict # # now parsing flags: # self.keep_inline_math = flags.pop('keep_inline_math', False) self.tolerant_parsing = flags.pop('tolerant_parsing', True) self.strict_braces = flags.pop('strict_braces', False) if flags: # any flags left which we haven't recognized logger.warning("LatexWalker(): Unknown flag(s) encountered: %r", flags.keys()) def parse_flags(self): return { 'keep_inline_math': self.keep_inline_math, 'tolerant_parsing': self.tolerant_parsing, } def get_token(self, pos, brackets_are_chars=True, environments=True, keep_inline_math=None): """ Parse the token in the stream pointed to at position `pos`. Returns a `LatexToken`. Raises `LatexWalkerEndOfStream` if end of stream reached. If `keep_inline_math` is not `None`, then that value overrides that of `self.keep_inline_math` for the duration of this method call. """ s = self.s # shorthand with _PushPropOverride(self, 'keep_inline_math', keep_inline_math): space = '' while (pos < len(s) and s[pos].isspace()): space += s[pos] pos += 1 if (space.endswith('\n\n')): # two \n's indicate new paragraph. # pre-space is overkill here I think. return LatexToken(tok='char', arg='\n\n', pos=pos-2, len=2, pre_space='') if (pos >= len(s)): raise LatexWalkerEndOfStream() if (s[pos] == '\\'): # escape sequence i = 2 macro = s[pos+1] # next char is necessarily part of macro # following chars part of macro only if all are alphabetical if (s[pos+1].isalpha()): while pos+i, pos, len)`. `pos` is the first char of the expression, and `len` is its length. """ with _PushPropOverride(self, 'strict_braces', strict_braces): tok = self.get_token(pos, environments=False, keep_inline_math=False) if (tok.tok == 'macro'): if (tok.arg == 'end'): if (not self.tolerant_parsing): # error, this should be an \end{environment}, not an argument in itself raise LatexWalkerParseError("Expected expression, got \end", self.s, pos) else: return (LatexCharsNode(chars=''), tok.pos, 0) return (LatexMacroNode(macroname=tok.arg, nodeoptarg=None, nodeargs=[]), tok.pos, tok.len) if (tok.tok == 'comment'): return self.get_latex_expression(pos+tok.len) if (tok.tok == 'brace_open'): return self.get_latex_braced_group(tok.pos) if (tok.tok == 'brace_close'): if (self.strict_braces and not self.tolerant_parsing): raise LatexWalkerParseError("Expected expression, got closing brace!", self.s, pos) return (LatexCharsNode(chars=''), tok.pos, 0) if (tok.tok == 'char'): return (LatexCharsNode(chars=tok.arg), tok.pos, tok.len) raise LatexWalkerParseError("Unknown token type: %s" %(tok.tok), self.s, pos) def get_latex_maybe_optional_arg(self, pos): """ Attempts to parse an optional argument. Returns a tuple `(groupnode, pos, len)` if success, otherwise returns None. """ tok = self.get_token(pos, brackets_are_chars=False, environments=False) if (tok.tok == 'brace_open' and tok.arg == '['): return self.get_latex_braced_group(pos, brace_type='[') return None def get_latex_braced_group(self, pos, brace_type='{'): """ Reads a latex expression enclosed in braces {...}. The first token of `s[pos:]` must be an opening brace. Returns a tuple `(node, pos, len)`. `pos` is the first char of the expression (which has to be an opening brace), and `len` is its length, including the closing brace. """ closing_brace = None if (brace_type == '{'): closing_brace = '}' elif (brace_type == '['): closing_brace = ']' else: raise LatexWalkerParseError(s=self.s, pos=pos, msg="Uknown brace type: %s" %(brace_type)) brackets_are_chars = (brace_type != '[') firsttok = self.get_token(pos, brackets_are_chars=brackets_are_chars) if (firsttok.tok != 'brace_open' or firsttok.arg != brace_type): raise LatexWalkerParseError(s=self.s, pos=pos, msg='get_latex_braced_group: not an opening brace/bracket: %s' %(self.s[pos])) #pos = firsttok.pos + firsttok.len (nodelist, npos, nlen) = self.get_latex_nodes(firsttok.pos + firsttok.len, stop_upon_closing_brace=closing_brace) return (LatexGroupNode(nodelist=nodelist), firsttok.pos, npos + nlen - firsttok.pos) def get_latex_environment(self, pos, environmentname=None): """ Reads a latex expression enclosed in a \\begin{environment}...\\end{environment}. The first token in the stream must be the \\begin{environment}. If `environmentname` is given and nonempty, then additionally a :py:exc:`LatexWalkerParseError` is raised if the environment in the input stream does not match the provided name. Returns a tuple (node, pos, len) with node being a :py:class:`LatexEnvironmentNode`. """ startpos = pos firsttok = self.get_token(pos) if (firsttok.tok != 'begin_environment' or (environmentname is not None and firsttok.arg != environmentname)): raise LatexWalkerParseError(s=self.s, pos=pos, msg=r'get_latex_environment: expected \begin{%s}: %s' %( environmentname if environmentname is not None else '', firsttok.arg )) if (environmentname is None): environmentname = firsttok.arg pos = firsttok.pos + firsttok.len optargs = [] args = [] # see if the \begin{environment} is immediately followed by some options. # BUG: Don't eat the brace of a commutator!! impose no space. optargtuple = None if (self.s[pos] == '['): optargtuple = self.get_latex_maybe_optional_arg(pos) if (optargtuple is not None): optargs.append(optargtuple[0]) pos = optargtuple[1]+optargtuple[2] else: # try to see if we have a mandatory argument # don't use get_token as we don't want to skip any space. if self.s[pos] == '{': (argnode, apos, alen) = self.get_latex_braced_group(pos) args.append(argnode) pos = apos+alen (nodelist, npos, nlen) = self.get_latex_nodes(pos, stop_upon_end_environment=environmentname) return (LatexEnvironmentNode(envname=environmentname, nodelist=nodelist, optargs=optargs, args=args), startpos, npos+nlen-startpos) def get_latex_nodes(self, pos=0, stop_upon_closing_brace=None, stop_upon_end_environment=None, stop_upon_closing_mathmode=None): """ Parses latex content stored in `self.s` into a list of nodes. Returns a tuple `(nodelist, pos, len)` where nodelist is a list of `LatexNode` 's. If `stop_upon_closing_brace` is given, then parsing stops once the given closing brace is encountered (but not inside a subgroup). The brace is given as a string, e.g. '}'. The returned `len` includes the closing brace, but the closing brace is not included in any of the nodes in the `nodelist`. If `stop_upon_end_environment` is provided, then parsing stops once the given environment was closed. If there is an environment mismatch, then a `LatexWalkerParseError` is raised except in tolerant parsing mode (see py:meth:`parse_flags()`). Again, the closing environment is included in the length count but not the nodes. """ nodelist = [] brackets_are_chars = True if (stop_upon_closing_brace == ']'): brackets_are_chars = False origpos = pos class PosPointer: def __init__(self, pos=0, lastchars=''): self.pos = pos self.lastchars = lastchars p = PosPointer(pos) def do_read(nodelist, p): """ Read a single token and process it, recursing into brace blocks and environments etc if needed, and appending stuff to nodelist. Return True whenever we should stop trying to read more. (e.g. upon reaching the a matched stop_upon_end_environment etc.) """ try: tok = self.get_token(p.pos, brackets_are_chars=brackets_are_chars) except LatexWalkerEndOfStream: if self.tolerant_parsing: return True raise # re-raise p.pos = tok.pos + tok.len # if it's a char, just append it to the stream of last characters. if (tok.tok == 'char'): p.lastchars += tok.pre_space + tok.arg return False # maybe add the pre_space of the new token to lastchars, if applicable. #if (len(tok.pre_space)): # p.lastchars += tok.pre_space # yields wayyy tooo much space in output!! # if it's not a char, push the last `p.lastchars` into the node list before anything else if (len(p.lastchars)): strnode = LatexCharsNode(chars=p.lastchars+tok.pre_space) nodelist.append(strnode) p.lastchars = '' # reset lastchars. # and see what the token is. if (tok.tok == 'brace_close'): # we've reached the end of the group. stop the parsing. if (tok.arg != stop_upon_closing_brace): if (not self.tolerant_parsing): raise LatexWalkerParseError(s=self.s, pos=tok.pos, msg='Unexpected mismatching closing brace: `%s\'' %(tok.arg)) return False return True if (tok.tok == 'end_environment'): # we've reached the end of an environment. if (tok.arg != stop_upon_end_environment): if (not self.tolerant_parsing): raise LatexWalkerParseError(s=self.s, pos=tok.pos, msg=('Unexpected mismatching closing environment: `%s\', ' 'expecting `%s\'' %(tok.arg, stop_upon_end_environment)) ) return False return True if (tok.tok == 'mathmode_inline'): # if we care about keeping math mode inlines verbatim, gulp all of the expression. if (stop_upon_closing_mathmode is not None): if (stop_upon_closing_mathmode != '$'): raise LatexWalkerParseError(s=self.s, pos=tok.pos, msg='Unexpected mismatching closing math mode: `$\'') return True # we have encountered a new math inline, so gulp all of the math expression (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes(p.pos, stop_upon_closing_mathmode='$') p.pos = mpos + mlen nodelist.append(LatexMathNode(displaytype='inline', nodelist=mathinline_nodelist)) return if (tok.tok == 'comment'): commentnode = LatexCommentNode(comment=tok.arg) nodelist.append(commentnode) return if (tok.tok == 'brace_open'): # another braced group to read. (groupnode, bpos, blen) = self.get_latex_braced_group(tok.pos) p.pos = bpos + blen nodelist.append(groupnode) return if (tok.tok == 'begin_environment'): # an environment to read. (envnode, epos, elen) = self.get_latex_environment(tok.pos, environmentname=tok.arg) p.pos = epos + elen # add node and continue. nodelist.append(envnode) return if (tok.tok == 'macro'): # read a macro. see if it has arguments. nodeoptarg = None nodeargs = [] macname = tok.arg.rstrip('*') if (macname in self.macro_dict): mac = self.macro_dict[macname] def getoptarg(pos): """Gets a possibly optional argument. returns (argnode, new-pos) where argnode might be `None` if the argument was not specified.""" optarginfotuple = self.get_latex_maybe_optional_arg(pos) if (optarginfotuple is not None): (nodeoptarg, optargpos, optarglen) = optarginfotuple return (nodeoptarg, optargpos+optarglen) return (None, pos) def getarg(pos): """Gets a mandatory argument. returns (argnode, new-pos)""" (nodearg, npos, nlen) = self.get_latex_expression(pos, strict_braces=False) return (nodearg, npos + nlen) if (mac.optarg): (nodeoptarg, p.pos) = getoptarg(p.pos) if (isinstance(mac.numargs, basestring)): # specific argument specification for arg in mac.numargs: if (arg == '{'): (node, p.pos) = getarg(p.pos) nodeargs.append(node) elif (arg == '['): (node, p.pos) = getoptarg(p.pos) nodeargs.append(node) else: raise LatexWalkerError("Unknown macro argument kind for macro %s: %s" % (mac.macroname, arg)) else: for n in range(mac.numargs): (nodearg, p.pos) = getarg(p.pos) nodeargs.append(nodearg) nodelist.append(LatexMacroNode(macroname=tok.arg, nodeoptarg=nodeoptarg, nodeargs=nodeargs)) return None raise LatexWalkerParseError(s=self.s, pos=p.pos, msg="Uknown token: %r" %(tok)) while True: try: r_endnow = do_read(nodelist, p) except LatexWalkerEndOfStream: if (stop_upon_closing_brace or stop_upon_end_environment): # unexpected eof if (not self.tolerant_parsing): raise LatexWalkerError("Unexpected end of stream!") else: r_endnow = False else: r_endnow = True if (r_endnow): # add last chars if (p.lastchars): strnode = LatexCharsNode(chars=p.lastchars) nodelist.append(strnode) return (nodelist, origpos, p.pos - origpos) raise LatexWalkerError("CONGRATULATIONS !! " "You are the first human to telepathically break an infinite loop !!!!!!!") # ------------------------------------------------------------------------------ def get_token(s, pos, brackets_are_chars=True, environments=True, **parse_flags): """ Parse the next token in the stream. Returns a `LatexToken`. Raises `LatexWalkerEndOfStream` if end of stream reached. .. deprecated:: 1.0 Please use :py:meth:`LatexWalker.get_token()` instead. """ return LatexWalker(s, **parse_flags).get_token(pos=pos, brackets_are_chars=brackets_are_chars, environments=environments) def get_latex_expression(s, pos, **parse_flags): """ Reads a latex expression, e.g. macro argument. This may be a single char, an escape sequence, or a expression placed in braces. Returns a tuple `(, pos, len)`. `pos` is the first char of the expression, and `len` is its length. .. deprecated:: 1.0 Please use :py:meth:`LatexWalker.get_latex_expression()` instead. """ return LatexWalker(s, **parse_flags).get_latex_expression(pos=pos) def get_latex_maybe_optional_arg(s, pos, **parse_flags): """ Attempts to parse an optional argument. Returns a tuple `(groupnode, pos, len)` if success, otherwise returns None. .. deprecated:: 1.0 Please use :py:meth:`LatexWalker.get_latex_maybe_optional_arg()` instead. """ return LatexWalker(s, **parse_flags).get_latex_maybe_optional_arg(pos=pos) def get_latex_braced_group(s, pos, brace_type='{', **parse_flags): """ Reads a latex expression enclosed in braces {...}. The first token of `s[pos:]` must be an opening brace. Returns a tuple `(node, pos, len)`. `pos` is the first char of the expression (which has to be an opening brace), and `len` is its length, including the closing brace. .. deprecated:: 1.0 Please use :py:meth:`LatexWalker.get_latex_braced_group()` instead. """ return LatexWalker(s, **parse_flags).get_latex_braced_group(pos=pos, brace_type=brace_type) def get_latex_environment(s, pos, environmentname=None, **parse_flags): """ Reads a latex expression enclosed in a \\begin{environment}...\\end{environment}. The first token in the stream must be the \\begin{environment}. Returns a tuple (node, pos, len) with node being a :py:class:`LatexEnvironmentNode`. .. deprecated:: 1.0 Please use :py:meth:`LatexWalker.get_latex_environment()` instead. """ return LatexWalker(s, **parse_flags).get_latex_environment(pos=pos, environmentname=environmentname) def get_latex_nodes(s, pos=0, stop_upon_closing_brace=None, stop_upon_end_environment=None, stop_upon_closing_mathmode=None, **parse_flags): """ Parses latex content `s`. Returns a tuple `(nodelist, pos, len)` where nodelist is a list of `LatexNode` 's. If `stop_upon_closing_brace` is given, then `len` includes the closing brace, but the closing brace is not included in any of the nodes in the `nodelist`. .. deprecated:: 1.0 Please use :py:meth:`LatexWalker.get_latex_nodes()` instead. """ return LatexWalker(s, **parse_flags).get_latex_nodes(stop_upon_closing_brace=stop_upon_closing_brace, stop_upon_end_environment=stop_upon_end_environment, stop_upon_closing_mathmode=stop_upon_closing_mathmode) # ------------------------------------------------------------------------------ # # small utilities for displaying & debugging # def put_in_braces(brace_char, thestring): if (brace_char == '{'): return '{%s}' %(thestring) if (brace_char == '['): return '[%s]' %(thestring) if (brace_char == '('): return '(%s)' %(thestring) if (brace_char == '<'): return '<%s>' %(thestring) return brace_char + thestring + brace_char def nodelist_to_latex(nodelist): latex = '' for n in nodelist: if n is None: continue if n.isNodeType(LatexCharsNode): latex += n.chars continue if n.isNodeType(LatexMacroNode): latex += r'\%s' %(n.macroname) mac = None if (n.macroname in default_macro_dict): mac = default_macro_dict[n.macroname] if (n.nodeoptarg is not None): latex += '[%s]' %(nodelist_to_latex([n.nodeoptarg])) if mac is not None: macbraces = (mac.numargs if isinstance(mac.numargs, basestring) else '{'*mac.numargs) else: macbraces = '{'*len(n.nodeargs) i = 0 if (len(n.nodeargs) != len(macbraces)): raise LatexWalkerError("Error: number of arguments (%d) provided to macro `\\%s' does not " "match its specification of `%s'" %(len(n.nodeargs), n.macroname, macbraces)) for i in range(len(n.nodeargs)): nodearg = n.nodeargs[i] if nodearg is not None: latex += put_in_braces(macbraces[i], nodelist_to_latex([nodearg])) continue if n.isNodeType(LatexCommentNode): latex += '%'+n.comment+'\n' continue if n.isNodeType(LatexGroupNode): latex += put_in_braces('{', nodelist_to_latex(n.nodelist)) continue if n.isNodeType(LatexEnvironmentNode): latex += r'\begin{%s}' %(n.envname) for optarg in n.optargs: latex += put_in_braces('[', nodelist_to_latex([optarg])) for arg in n.args: latex += put_in_braces('{', nodelist_to_latex([arg])) latex += nodelist_to_latex(n.nodelist) latex += r'\end{%s}' %(n.envname) continue latex += '<[UNKNOWN LATEX NODE: `%s\']>' %(n.nodeType().__name__) return latex def math_node_to_latex(node): if (not node.isNodeType(LatexMathNode)): raise LatexWalkerError("Expected math node, got `%s'" %(node.nodeType().__name__)) if (node.displaytype == 'inline'): return '$%s$' %(nodelist_to_latex(node.nodelist)) if (node.displaytype == 'display'): return r'\[%s\]' %(nodelist_to_latex(node.nodelist)) raise LatexWalkerError("Unkonwn displaytype: `%s'" %(node.displaytype)) def disp_node(n, indent=0, context='* ', skip_group=False): title = '' comment = '' iterchildren = [] if n is None: title = '' elif n.isNodeType(LatexCharsNode): title = "'%s'" %(n.chars.strip()) elif n.isNodeType(LatexMacroNode): title = '\\'+n.macroname #comment = 'opt arg?: %d; %d args' % (n.arg.nodeoptarg is not None, len(n.arg.nodeargs)) if (n.nodeoptarg): iterchildren.append(('[...]: ', [n.nodeoptarg], False)) if (len(n.nodeargs)): iterchildren.append(('{...}: ', n.nodeargs, False)) elif n.isNodeType(LatexCommentNode): title = '%' + n.comment.strip() elif n.isNodeType(LatexGroupNode): if (skip_group): for nn in n.arg: disp_node(nn, indent=indent, context=context) return title = 'Group: ' iterchildren.append(('* ', n.nodelist, False)) elif n.isNodeType(LatexEnvironmentNode): title = '\\begin{%s}' %(n.envname) iterchildren.append(('* ', n.nodelist, False)) else: print("UNKNOWN NODE TYPE: %s"%(n.nodeType().__name__)) print(' '*indent + context + title + ' '+comment) for context, nodelist, skip in iterchildren: for nn in nodelist: disp_node(nn, indent=indent+4, context=context, skip_group=skip) if __name__ == '__main__': try: #latex = '\\textit{hi there!} This is {\em an equation}: \\begin{equation}\n a + bi = 0\n\\end{equation}\n\nwhere $i$ is the imaginary unit.\n' #nodelist = get_latex_nodes_debug(latex) #print repr(nodelist) import fileinput latex = '' for line in fileinput.input(): latex += line (nodes, pos, llen) = get_latex_nodes(latex) print('\n--- NODES ---\n') print(repr(nodes)) print('\n-------------\n') print('\n--- NODES ---\n') for n in nodes: disp_node(n) print('\n-------------\n') except: import pdb import sys print("\nEXCEPTION: " + unicode(sys.exc_info()[1]) + "\n") pdb.post_mortem()