Back to index

enigmail  1.4.3
parser.py
Go to the documentation of this file.
00001 """
00002 Module for parsing Makefile syntax.
00003 
00004 Makefiles use a line-based parsing system. Continuations and substitutions are handled differently based on the
00005 type of line being parsed:
00006 
00007 Lines with makefile syntax condense continuations to a single space, no matter the actual trailing whitespace
00008 of the first line or the leading whitespace of the continuation. In other situations, trailing whitespace is
00009 relevant.
00010 
00011 Lines with command syntax do not condense continuations: the backslash and newline are part of the command.
00012 (GNU Make is buggy in this regard, at least on mac).
00013 
00014 Lines with an initial tab are commands if they can be (there is a rule or a command immediately preceding).
00015 Otherwise, they are parsed as makefile syntax.
00016 
00017 This file parses into the data structures defined in the parserdata module. Those classes are what actually
00018 do the dirty work of "executing" the parsed data into a data.Makefile.
00019 
00020 Four iterator functions are available:
00021 * iterdata
00022 * itermakefilechars
00023 * itercommandchars
00024 
00025 The iterators handle line continuations and comments in different ways, but share a common calling
00026 convention:
00027 
00028 Called with (data, startoffset, tokenlist, finditer)
00029 
00030 yield 4-tuples (flatstr, token, tokenoffset, afteroffset)
00031 flatstr is data, guaranteed to have no tokens (may be '')
00032 token, tokenoffset, afteroffset *may be None*. That means there is more text
00033 coming.
00034 """
00035 
00036 import logging, re, os, sys
00037 import data, functions, util, parserdata
00038 
00039 _log = logging.getLogger('pymake.parser')
00040 
00041 class SyntaxError(util.MakeError):
00042     pass
00043 
00044 _skipws = re.compile('\S')
00045 class Data(object):
00046     """
00047     A single virtual "line", which can be multiple source lines joined with
00048     continuations.
00049     """
00050 
00051     __slots__ = ('s', 'lstart', 'lend', 'loc')
00052 
00053     def __init__(self, s, lstart, lend, loc):
00054         self.s = s
00055         self.lstart = lstart
00056         self.lend = lend
00057         self.loc = loc
00058 
00059     @staticmethod
00060     def fromstring(s, path):
00061         return Data(s, 0, len(s), parserdata.Location(path, 1, 0))
00062 
00063     def getloc(self, offset):
00064         assert offset >= self.lstart and offset <= self.lend
00065         return self.loc.offset(self.s, self.lstart, offset)
00066 
00067     def skipwhitespace(self, offset):
00068         """
00069         Return the offset of the first non-whitespace character in data starting at offset, or None if there are
00070         only whitespace characters remaining.
00071         """
00072         m = _skipws.search(self.s, offset, self.lend)
00073         if m is None:
00074             return self.lend
00075 
00076         return m.start(0)
00077 
00078 _linere = re.compile(r'\\*\n')
00079 def enumeratelines(s, filename):
00080     """
00081     Enumerate lines in a string as Data objects, joining line
00082     continuations.
00083     """
00084 
00085     off = 0
00086     lineno = 1
00087     curlines = 0
00088     for m in _linere.finditer(s):
00089         curlines += 1
00090         start, end = m.span(0)
00091 
00092         if (start - end) % 2 == 0:
00093             # odd number of backslashes is a continuation
00094             continue
00095 
00096         yield Data(s, off, end - 1, parserdata.Location(filename, lineno, 0))
00097 
00098         lineno += curlines
00099         curlines = 0
00100         off = end
00101 
00102     yield Data(s, off, len(s), parserdata.Location(filename, lineno, 0))
00103 
00104 _alltokens = re.compile(r'''\\*\# | # hash mark preceeded by any number of backslashes
00105                             := |
00106                             \+= |
00107                             \?= |
00108                             :: |
00109                             (?:\$(?:$|[\(\{](?:%s)\s+|.)) | # dollar sign followed by EOF, a function keyword with whitespace, or any character
00110                             :(?![\\/]) | # colon followed by anything except a slash (Windows path detection)
00111                             [=#{}();,|'"]''' % '|'.join(functions.functionmap.iterkeys()), re.VERBOSE)
00112 
00113 def iterdata(d, offset, tokenlist, it):
00114     """
00115     Iterate over flat data without line continuations, comments, or any special escaped characters.
00116 
00117     Typically used to parse recursively-expanded variables.
00118     """
00119 
00120     assert len(tokenlist), "Empty tokenlist passed to iterdata is meaningless!"
00121     assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
00122 
00123     if offset == d.lend:
00124         return
00125 
00126     s = d.s
00127     for m in it:
00128         mstart, mend = m.span(0)
00129         token = s[mstart:mend]
00130         if token in tokenlist or (token[0] == '$' and '$' in tokenlist):
00131             yield s[offset:mstart], token, mstart, mend
00132         else:
00133             yield s[offset:mend], None, None, mend
00134         offset = mend
00135 
00136     yield s[offset:d.lend], None, None, None
00137 
00138 # multiple backslashes before a newline are unescaped, halving their total number
00139 _makecontinuations = re.compile(r'(?:\s*|((?:\\\\)+))\\\n\s*')
00140 def _replacemakecontinuations(m):
00141     start, end = m.span(1)
00142     if start == -1:
00143         return ' '
00144     return ' '.rjust((end - start) / 2 + 1, '\\')
00145 
00146 def itermakefilechars(d, offset, tokenlist, it, ignorecomments=False):
00147     """
00148     Iterate over data in makefile syntax. Comments are found at unescaped # characters, and escaped newlines
00149     are converted to single-space continuations.
00150     """
00151 
00152     assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
00153 
00154     if offset == d.lend:
00155         return
00156 
00157     s = d.s
00158     for m in it:
00159         mstart, mend = m.span(0)
00160         token = s[mstart:mend]
00161 
00162         starttext = _makecontinuations.sub(_replacemakecontinuations, s[offset:mstart])
00163 
00164         if token[-1] == '#' and not ignorecomments:
00165             l = mend - mstart
00166             # multiple backslashes before a hash are unescaped, halving their total number
00167             if l % 2:
00168                 # found a comment
00169                 yield starttext + token[:(l - 1) / 2], None, None, None
00170                 return
00171             else:
00172                 yield starttext + token[-l / 2:], None, None, mend
00173         elif token in tokenlist or (token[0] == '$' and '$' in tokenlist):
00174             yield starttext, token, mstart, mend
00175         else:
00176             yield starttext + token, None, None, mend
00177         offset = mend
00178 
00179     yield _makecontinuations.sub(_replacemakecontinuations, s[offset:d.lend]), None, None, None
00180 
00181 _findcomment = re.compile(r'\\*\#')
00182 def flattenmakesyntax(d, offset):
00183     """
00184     A shortcut method for flattening line continuations and comments in makefile syntax without
00185     looking for other tokens.
00186     """
00187 
00188     assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
00189     if offset == d.lend:
00190         return ''
00191 
00192     s = _makecontinuations.sub(_replacemakecontinuations, d.s[offset:d.lend])
00193 
00194     elements = []
00195     offset = 0
00196     for m in _findcomment.finditer(s):
00197         mstart, mend = m.span(0)
00198         elements.append(s[offset:mstart])
00199         if (mend - mstart) % 2:
00200             # even number of backslashes... it's a comment
00201             elements.append(''.ljust((mend - mstart - 1) / 2, '\\'))
00202             return ''.join(elements)
00203 
00204         # odd number of backslashes
00205         elements.append(''.ljust((mend - mstart - 2) / 2, '\\') + '#')
00206         offset = mend
00207 
00208     elements.append(s[offset:])
00209     return ''.join(elements)
00210 
00211 def itercommandchars(d, offset, tokenlist, it):
00212     """
00213     Iterate over command syntax. # comment markers are not special, and escaped newlines are included
00214     in the output text.
00215     """
00216 
00217     assert offset >= d.lstart and offset <= d.lend, "offset %i should be between %i and %i" % (offset, d.lstart, d.lend)
00218 
00219     if offset == d.lend:
00220         return
00221 
00222     s = d.s
00223     for m in it:
00224         mstart, mend = m.span(0)
00225         token = s[mstart:mend]
00226         starttext = s[offset:mstart].replace('\n\t', '\n')
00227 
00228         if token in tokenlist or (token[0] == '$' and '$' in tokenlist):
00229             yield starttext, token, mstart, mend
00230         else:
00231             yield starttext + token, None, None, mend
00232         offset = mend
00233 
00234     yield s[offset:d.lend].replace('\n\t', '\n'), None, None, None
00235 
00236 _redefines = re.compile('\s*define|\s*endef')
00237 def iterdefinelines(it, startloc):
00238     """
00239     Process the insides of a define. Most characters are included literally. Escaped newlines are treated
00240     as they would be in makefile syntax. Internal define/endef pairs are ignored.
00241     """
00242 
00243     results = []
00244 
00245     definecount = 1
00246     for d in it:
00247         m = _redefines.match(d.s, d.lstart, d.lend)
00248         if m is not None:
00249             directive = m.group(0).strip()
00250             if directive == 'endef':
00251                 definecount -= 1
00252                 if definecount == 0:
00253                     return _makecontinuations.sub(_replacemakecontinuations, '\n'.join(results))
00254             else:
00255                 definecount += 1
00256 
00257         results.append(d.s[d.lstart:d.lend])
00258 
00259     # Falling off the end is an unterminated define!
00260     raise SyntaxError("define without matching endef", startloc)
00261 
00262 def _ensureend(d, offset, msg):
00263     """
00264     Ensure that only whitespace remains in this data.
00265     """
00266 
00267     s = flattenmakesyntax(d, offset)
00268     if s != '' and not s.isspace():
00269         raise SyntaxError(msg, d.getloc(offset))
00270 
00271 _eqargstokenlist = ('(', "'", '"')
00272 
00273 def ifeq(d, offset):
00274     if offset > d.lend - 1:
00275         raise SyntaxError("No arguments after conditional", d.getloc(offset))
00276 
00277     # the variety of formats for this directive is rather maddening
00278     token = d.s[offset]
00279     if token not in _eqargstokenlist:
00280         raise SyntaxError("No arguments after conditional", d.getloc(offset))
00281 
00282     offset += 1
00283 
00284     if token == '(':
00285         arg1, t, offset = parsemakesyntax(d, offset, (',',), itermakefilechars)
00286         if t is None:
00287             raise SyntaxError("Expected two arguments in conditional", d.getloc(d.lend))
00288 
00289         arg1.rstrip()
00290 
00291         offset = d.skipwhitespace(offset)
00292         arg2, t, offset = parsemakesyntax(d, offset, (')',), itermakefilechars)
00293         if t is None:
00294             raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
00295 
00296         _ensureend(d, offset, "Unexpected text after conditional")
00297     else:
00298         arg1, t, offset = parsemakesyntax(d, offset, (token,), itermakefilechars)
00299         if t is None:
00300             raise SyntaxError("Unexpected text in conditional", d.getloc(d.lend))
00301 
00302         offset = d.skipwhitespace(offset)
00303         if offset == d.lend:
00304             raise SyntaxError("Expected two arguments in conditional", d.getloc(offset))
00305 
00306         token = d.s[offset]
00307         if token not in '\'"':
00308             raise SyntaxError("Unexpected text in conditional", d.getloc(offset))
00309 
00310         arg2, t, offset = parsemakesyntax(d, offset + 1, (token,), itermakefilechars)
00311 
00312         _ensureend(d, offset, "Unexpected text after conditional")
00313 
00314     return parserdata.EqCondition(arg1, arg2)
00315 
00316 def ifneq(d, offset):
00317     c = ifeq(d, offset)
00318     c.expected = False
00319     return c
00320 
00321 def ifdef(d, offset):
00322     e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
00323     e.rstrip()
00324 
00325     return parserdata.IfdefCondition(e)
00326 
00327 def ifndef(d, offset):
00328     c = ifdef(d, offset)
00329     c.expected = False
00330     return c
00331 
00332 _conditionkeywords = {
00333     'ifeq': ifeq,
00334     'ifneq': ifneq,
00335     'ifdef': ifdef,
00336     'ifndef': ifndef
00337     }
00338 
00339 _conditiontokens = tuple(_conditionkeywords.iterkeys())
00340 _conditionre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_conditiontokens))
00341 
00342 _directivestokenlist = _conditiontokens + \
00343     ('else', 'endif', 'define', 'endef', 'override', 'include', '-include', 'includedeps', '-includedeps', 'vpath', 'export', 'unexport')
00344 
00345 _directivesre = re.compile(r'(%s)(?:$|\s+)' % '|'.join(_directivestokenlist))
00346 
00347 _varsettokens = (':=', '+=', '?=', '=')
00348 
00349 def _parsefile(pathname):
00350     fd = open(pathname, "rU")
00351     stmts = parsestring(fd.read(), pathname)
00352     stmts.mtime = os.fstat(fd.fileno()).st_mtime
00353     fd.close()
00354     return stmts
00355 
00356 def _checktime(path, stmts):
00357     mtime = os.path.getmtime(path)
00358     if mtime != stmts.mtime:
00359         _log.debug("Re-parsing makefile '%s': mtimes differ", path)
00360         return False
00361 
00362     return True
00363 
00364 _parsecache = util.MostUsedCache(15, _parsefile, _checktime)
00365 
00366 def parsefile(pathname):
00367     """
00368     Parse a filename into a parserdata.StatementList. A cache is used to avoid re-parsing
00369     makefiles that have already been parsed and have not changed.
00370     """
00371 
00372     pathname = os.path.realpath(pathname)
00373     return _parsecache.get(pathname)
00374 
00375 def parsestring(s, filename):
00376     """
00377     Parse a string containing makefile data into a parserdata.StatementList.
00378     """
00379 
00380     currule = False
00381     condstack = [parserdata.StatementList()]
00382 
00383     fdlines = enumeratelines(s, filename)
00384     for d in fdlines:
00385         assert len(condstack) > 0
00386 
00387         offset = d.lstart
00388 
00389         if currule and offset < d.lend and d.s[offset] == '\t':
00390             e, token, offset = parsemakesyntax(d, offset + 1, (), itercommandchars)
00391             assert token is None
00392             assert offset is None
00393             condstack[-1].append(parserdata.Command(e))
00394             continue
00395 
00396         # To parse Makefile syntax, we first strip leading whitespace and
00397         # look for initial keywords. If there are no keywords, it's either
00398         # setting a variable or writing a rule.
00399 
00400         offset = d.skipwhitespace(offset)
00401         if offset is None:
00402             continue
00403 
00404         m = _directivesre.match(d.s, offset, d.lend)
00405         if m is not None:
00406             kword = m.group(1)
00407             offset = m.end(0)
00408 
00409             if kword == 'endif':
00410                 _ensureend(d, offset, "Unexpected data after 'endif' directive")
00411                 if len(condstack) == 1:
00412                     raise SyntaxError("unmatched 'endif' directive",
00413                                       d.getloc(offset))
00414 
00415                 condstack.pop().endloc = d.getloc(offset)
00416                 continue
00417             
00418             if kword == 'else':
00419                 if len(condstack) == 1:
00420                     raise SyntaxError("unmatched 'else' directive",
00421                                       d.getloc(offset))
00422 
00423                 m = _conditionre.match(d.s, offset, d.lend)
00424                 if m is None:
00425                     _ensureend(d, offset, "Unexpected data after 'else' directive.")
00426                     condstack[-1].addcondition(d.getloc(offset), parserdata.ElseCondition())
00427                 else:
00428                     kword = m.group(1)
00429                     if kword not in _conditionkeywords:
00430                         raise SyntaxError("Unexpected condition after 'else' directive.",
00431                                           d.getloc(offset))
00432 
00433                     startoffset = offset
00434                     offset = d.skipwhitespace(m.end(1))
00435                     c = _conditionkeywords[kword](d, offset)
00436                     condstack[-1].addcondition(d.getloc(startoffset), c)
00437                 continue
00438 
00439             if kword in _conditionkeywords:
00440                 c = _conditionkeywords[kword](d, offset)
00441                 cb = parserdata.ConditionBlock(d.getloc(d.lstart), c)
00442                 condstack[-1].append(cb)
00443                 condstack.append(cb)
00444                 continue
00445 
00446             if kword == 'endef':
00447                 raise SyntaxError("endef without matching define", d.getloc(offset))
00448 
00449             if kword == 'define':
00450                 currule = False
00451                 vname, t, i = parsemakesyntax(d, offset, (), itermakefilechars)
00452                 vname.rstrip()
00453 
00454                 startloc = d.getloc(d.lstart)
00455                 value = iterdefinelines(fdlines, startloc)
00456                 condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=startloc, token='=', targetexp=None))
00457                 continue
00458 
00459             if kword in ('include', '-include', 'includedeps', '-includedeps'):
00460                 if kword.startswith('-'):
00461                     required = False
00462                     kword = kword[1:]
00463                 else:
00464                     required = True
00465 
00466                 deps = kword == 'includedeps'
00467 
00468                 currule = False
00469                 incfile, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
00470                 condstack[-1].append(parserdata.Include(incfile, required, deps))
00471 
00472                 continue
00473 
00474             if kword == 'vpath':
00475                 currule = False
00476                 e, t, offset = parsemakesyntax(d, offset, (), itermakefilechars)
00477                 condstack[-1].append(parserdata.VPathDirective(e))
00478                 continue
00479 
00480             if kword == 'override':
00481                 currule = False
00482                 vname, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars)
00483                 vname.lstrip()
00484                 vname.rstrip()
00485 
00486                 if token is None:
00487                     raise SyntaxError("Malformed override directive, need =", d.getloc(d.lstart))
00488 
00489                 value = flattenmakesyntax(d, offset).lstrip()
00490 
00491                 condstack[-1].append(parserdata.SetVariable(vname, value=value, valueloc=d.getloc(offset), token=token, targetexp=None, source=data.Variables.SOURCE_OVERRIDE))
00492                 continue
00493 
00494             if kword == 'export':
00495                 currule = False
00496                 e, token, offset = parsemakesyntax(d, offset, _varsettokens, itermakefilechars)
00497                 e.lstrip()
00498                 e.rstrip()
00499 
00500                 if token is None:
00501                     condstack[-1].append(parserdata.ExportDirective(e, single=False))
00502                 else:
00503                     condstack[-1].append(parserdata.ExportDirective(e, single=True))
00504 
00505                     value = flattenmakesyntax(d, offset).lstrip()
00506                     condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None))
00507 
00508                 continue
00509 
00510             if kword == 'unexport':
00511                 e, token, offset = parsemakesyntax(d, offset, (), itermakefilechars)
00512                 condstack[-1].append(parserdata.UnexportDirective(e))
00513                 continue
00514 
00515         e, token, offset = parsemakesyntax(d, offset, _varsettokens + ('::', ':'), itermakefilechars)
00516         if token is None:
00517             e.rstrip()
00518             e.lstrip()
00519             if not e.isempty():
00520                 condstack[-1].append(parserdata.EmptyDirective(e))
00521             continue
00522 
00523         # if we encountered real makefile syntax, the current rule is over
00524         currule = False
00525 
00526         if token in _varsettokens:
00527             e.lstrip()
00528             e.rstrip()
00529 
00530             value = flattenmakesyntax(d, offset).lstrip()
00531 
00532             condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=None))
00533         else:
00534             doublecolon = token == '::'
00535 
00536             # `e` is targets or target patterns, which can end up as
00537             # * a rule
00538             # * an implicit rule
00539             # * a static pattern rule
00540             # * a target-specific variable definition
00541             # * a pattern-specific variable definition
00542             # any of the rules may have order-only prerequisites
00543             # delimited by |, and a command delimited by ;
00544             targets = e
00545 
00546             e, token, offset = parsemakesyntax(d, offset,
00547                                                _varsettokens + (':', '|', ';'),
00548                                                itermakefilechars)
00549             if token in (None, ';'):
00550                 condstack[-1].append(parserdata.Rule(targets, e, doublecolon))
00551                 currule = True
00552 
00553                 if token == ';':
00554                     offset = d.skipwhitespace(offset)
00555                     e, t, offset = parsemakesyntax(d, offset, (), itercommandchars)
00556                     condstack[-1].append(parserdata.Command(e))
00557 
00558             elif token in _varsettokens:
00559                 e.lstrip()
00560                 e.rstrip()
00561 
00562                 value = flattenmakesyntax(d, offset).lstrip()
00563                 condstack[-1].append(parserdata.SetVariable(e, value=value, valueloc=d.getloc(offset), token=token, targetexp=targets))
00564             elif token == '|':
00565                 raise SyntaxError('order-only prerequisites not implemented', d.getloc(offset))
00566             else:
00567                 assert token == ':'
00568                 # static pattern rule
00569 
00570                 pattern = e
00571 
00572                 deps, token, offset = parsemakesyntax(d, offset, (';',), itermakefilechars)
00573 
00574                 condstack[-1].append(parserdata.StaticPatternRule(targets, pattern, deps, doublecolon))
00575                 currule = True
00576 
00577                 if token == ';':
00578                     offset = d.skipwhitespace(offset)
00579                     e, token, offset = parsemakesyntax(d, offset, (), itercommandchars)
00580                     condstack[-1].append(parserdata.Command(e))
00581 
00582     if len(condstack) != 1:
00583         raise SyntaxError("Condition never terminated with endif", condstack[-1].loc)
00584 
00585     return condstack[0]
00586 
00587 _PARSESTATE_TOPLEVEL = 0    # at the top level
00588 _PARSESTATE_FUNCTION = 1    # expanding a function call
00589 _PARSESTATE_VARNAME = 2     # expanding a variable expansion.
00590 _PARSESTATE_SUBSTFROM = 3   # expanding a variable expansion substitution "from" value
00591 _PARSESTATE_SUBSTTO = 4     # expanding a variable expansion substitution "to" value
00592 _PARSESTATE_PARENMATCH = 5  # inside nested parentheses/braces that must be matched
00593 
00594 class ParseStackFrame(object):
00595     __slots__ = ('parsestate', 'parent', 'expansion', 'tokenlist', 'openbrace', 'closebrace', 'function', 'loc', 'varname', 'substfrom')
00596 
00597     def __init__(self, parsestate, parent, expansion, tokenlist, openbrace, closebrace, function=None, loc=None):
00598         self.parsestate = parsestate
00599         self.parent = parent
00600         self.expansion = expansion
00601         self.tokenlist = tokenlist
00602         self.openbrace = openbrace
00603         self.closebrace = closebrace
00604         self.function = function
00605         self.loc = loc
00606 
00607     def __str__(self):
00608         return "<state=%i expansion=%s tokenlist=%s openbrace=%s closebrace=%s>" % (self.parsestate, self.expansion, self.tokenlist, self.openbrace, self.closebrace)
00609 
00610 _matchingbrace = {
00611     '(': ')',
00612     '{': '}',
00613     }
00614 
00615 def parsemakesyntax(d, offset, stopon, iterfunc):
00616     """
00617     Given Data, parse it into a data.Expansion.
00618 
00619     @param stopon (sequence)
00620         Indicate characters where toplevel parsing should stop.
00621 
00622     @param iterfunc (generator function)
00623         A function which is used to iterate over d, yielding (char, offset, loc)
00624         @see iterdata
00625         @see itermakefilechars
00626         @see itercommandchars
00627  
00628     @return a tuple (expansion, token, offset). If all the data is consumed,
00629     token and offset will be None
00630     """
00631 
00632     assert callable(iterfunc)
00633 
00634     stacktop = ParseStackFrame(_PARSESTATE_TOPLEVEL, None, data.Expansion(loc=d.getloc(d.lstart)),
00635                                tokenlist=stopon + ('$',),
00636                                openbrace=None, closebrace=None)
00637 
00638     tokeniterator = _alltokens.finditer(d.s, offset, d.lend)
00639 
00640     di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
00641     while True: # this is not a for loop because `di` changes during the function
00642         assert stacktop is not None
00643         try:
00644             s, token, tokenoffset, offset = di.next()
00645         except StopIteration:
00646             break
00647 
00648         stacktop.expansion.appendstr(s)
00649         if token is None:
00650             continue
00651 
00652         parsestate = stacktop.parsestate
00653 
00654         if token[0] == '$':
00655             if tokenoffset + 1 == d.lend:
00656                 # an unterminated $ expands to nothing
00657                 break
00658 
00659             loc = d.getloc(tokenoffset)
00660             c = token[1]
00661             if c == '$':
00662                 assert len(token) == 2
00663                 stacktop.expansion.appendstr('$')
00664             elif c in ('(', '{'):
00665                 closebrace = _matchingbrace[c]
00666 
00667                 if len(token) > 2:
00668                     fname = token[2:].rstrip()
00669                     fn = functions.functionmap[fname](loc)
00670                     e = data.Expansion()
00671                     if len(fn) + 1 == fn.maxargs:
00672                         tokenlist = (c, closebrace, '$')
00673                     else:
00674                         tokenlist = (',', c, closebrace, '$')
00675 
00676                     stacktop = ParseStackFrame(_PARSESTATE_FUNCTION, stacktop,
00677                                                e, tokenlist, function=fn,
00678                                                openbrace=c, closebrace=closebrace)
00679                 else:
00680                     e = data.Expansion()
00681                     tokenlist = (':', c, closebrace, '$')
00682                     stacktop = ParseStackFrame(_PARSESTATE_VARNAME, stacktop,
00683                                                e, tokenlist,
00684                                                openbrace=c, closebrace=closebrace, loc=loc)
00685             else:
00686                 assert len(token) == 2
00687                 e = data.Expansion.fromstring(c, loc)
00688                 stacktop.expansion.appendfunc(functions.VariableRef(loc, e))
00689         elif token in ('(', '{'):
00690             assert token == stacktop.openbrace
00691 
00692             stacktop.expansion.appendstr(token)
00693             stacktop = ParseStackFrame(_PARSESTATE_PARENMATCH, stacktop,
00694                                        stacktop.expansion,
00695                                        (token, stacktop.closebrace, '$'),
00696                                        openbrace=token, closebrace=stacktop.closebrace, loc=d.getloc(tokenoffset))
00697         elif parsestate == _PARSESTATE_PARENMATCH:
00698             assert token == stacktop.closebrace
00699             stacktop.expansion.appendstr(token)
00700             stacktop = stacktop.parent
00701         elif parsestate == _PARSESTATE_TOPLEVEL:
00702             assert stacktop.parent is None
00703             return stacktop.expansion.finish(), token, offset
00704         elif parsestate == _PARSESTATE_FUNCTION:
00705             if token == ',':
00706                 stacktop.function.append(stacktop.expansion.finish())
00707 
00708                 stacktop.expansion = data.Expansion()
00709                 if len(stacktop.function) + 1 == stacktop.function.maxargs:
00710                     tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
00711                     stacktop.tokenlist = tokenlist
00712             elif token in (')', '}'):
00713                 fn = stacktop.function
00714                 fn.append(stacktop.expansion.finish())
00715                 fn.setup()
00716                 
00717                 stacktop = stacktop.parent
00718                 stacktop.expansion.appendfunc(fn)
00719             else:
00720                 assert False, "Not reached, _PARSESTATE_FUNCTION"
00721         elif parsestate == _PARSESTATE_VARNAME:
00722             if token == ':':
00723                 stacktop.varname = stacktop.expansion
00724                 stacktop.parsestate = _PARSESTATE_SUBSTFROM
00725                 stacktop.expansion = data.Expansion()
00726                 stacktop.tokenlist = ('=', stacktop.openbrace, stacktop.closebrace, '$')
00727             elif token in (')', '}'):
00728                 fn = functions.VariableRef(stacktop.loc, stacktop.expansion.finish())
00729                 stacktop = stacktop.parent
00730                 stacktop.expansion.appendfunc(fn)
00731             else:
00732                 assert False, "Not reached, _PARSESTATE_VARNAME"
00733         elif parsestate == _PARSESTATE_SUBSTFROM:
00734             if token == '=':
00735                 stacktop.substfrom = stacktop.expansion
00736                 stacktop.parsestate = _PARSESTATE_SUBSTTO
00737                 stacktop.expansion = data.Expansion()
00738                 stacktop.tokenlist = (stacktop.openbrace, stacktop.closebrace, '$')
00739             elif token in (')', '}'):
00740                 # A substitution of the form $(VARNAME:.ee) is probably a mistake, but make
00741                 # parses it. Issue a warning. Combine the varname and substfrom expansions to
00742                 # make the compatible varname. See tests/var-substitutions.mk SIMPLE3SUBSTNAME
00743                 _log.warning("%s: Variable reference looks like substitution without =", stacktop.loc)
00744                 stacktop.varname.appendstr(':')
00745                 stacktop.varname.concat(stacktop.expansion)
00746                 fn = functions.VariableRef(stacktop.loc, stacktop.varname.finish())
00747                 stacktop = stacktop.parent
00748                 stacktop.expansion.appendfunc(fn)
00749             else:
00750                 assert False, "Not reached, _PARSESTATE_SUBSTFROM"
00751         elif parsestate == _PARSESTATE_SUBSTTO:
00752             assert token in  (')','}'), "Not reached, _PARSESTATE_SUBSTTO"
00753 
00754             fn = functions.SubstitutionRef(stacktop.loc, stacktop.varname.finish(),
00755                                            stacktop.substfrom.finish(), stacktop.expansion.finish())
00756             stacktop = stacktop.parent
00757             stacktop.expansion.appendfunc(fn)
00758         else:
00759             assert False, "Unexpected parse state %s" % stacktop.parsestate
00760 
00761         if stacktop.parent is not None and iterfunc == itercommandchars:
00762             di = itermakefilechars(d, offset, stacktop.tokenlist, tokeniterator,
00763                                    ignorecomments=True)
00764         else:
00765             di = iterfunc(d, offset, stacktop.tokenlist, tokeniterator)
00766 
00767     if stacktop.parent is not None:
00768         raise SyntaxError("Unterminated function call", d.getloc(offset))
00769 
00770     assert stacktop.parsestate == _PARSESTATE_TOPLEVEL
00771 
00772     return stacktop.expansion.finish(), None, None