Back to index

enigmail  1.4.3
lex.py
Go to the documentation of this file.
00001 # -----------------------------------------------------------------------------
00002 # ply: lex.py
00003 #
00004 # Copyright (C) 2001-2009,
00005 # David M. Beazley (Dabeaz LLC)
00006 # All rights reserved.
00007 #
00008 # Redistribution and use in source and binary forms, with or without
00009 # modification, are permitted provided that the following conditions are
00010 # met:
00011 # 
00012 # * Redistributions of source code must retain the above copyright notice,
00013 #   this list of conditions and the following disclaimer.  
00014 # * Redistributions in binary form must reproduce the above copyright notice, 
00015 #   this list of conditions and the following disclaimer in the documentation
00016 #   and/or other materials provided with the distribution.  
00017 # * Neither the name of the David Beazley or Dabeaz LLC may be used to
00018 #   endorse or promote products derived from this software without
00019 #  specific prior written permission. 
00020 #
00021 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
00022 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
00023 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
00024 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
00025 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
00027 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
00028 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
00029 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
00030 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
00031 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 # -----------------------------------------------------------------------------
00033 
00034 __version__    = "3.3"
00035 __tabversion__ = "3.2"       # Version of table file used
00036 
00037 import re, sys, types, copy, os
00038 
00039 # This tuple contains known string types
00040 try:
00041     # Python 2.6
00042     StringTypes = (types.StringType, types.UnicodeType)
00043 except AttributeError:
00044     # Python 3.0
00045     StringTypes = (str, bytes)
00046 
00047 # Extract the code attribute of a function. Different implementations
00048 # are for Python 2/3 compatibility.
00049 
00050 if sys.version_info[0] < 3:
00051     def func_code(f):
00052         return f.func_code
00053 else:
00054     def func_code(f):
00055         return f.__code__
00056 
00057 # This regular expression is used to match valid token names
00058 _is_identifier = re.compile(r'^[a-zA-Z0-9_]+$')
00059 
00060 # Exception thrown when invalid token encountered and no default error
00061 # handler is defined.
00062 
00063 class LexError(Exception):
00064     def __init__(self,message,s):
00065          self.args = (message,)
00066          self.text = s
00067 
00068 # Token class.  This class is used to represent the tokens produced.
00069 class LexToken(object):
00070     def __str__(self):
00071         return "LexToken(%s,%r,%d,%d)" % (self.type,self.value,self.lineno,self.lexpos)
00072     def __repr__(self):
00073         return str(self)
00074 
00075 # This object is a stand-in for a logging object created by the 
00076 # logging module.  
00077 
00078 class PlyLogger(object):
00079     def __init__(self,f):
00080         self.f = f
00081     def critical(self,msg,*args,**kwargs):
00082         self.f.write((msg % args) + "\n")
00083 
00084     def warning(self,msg,*args,**kwargs):
00085         self.f.write("WARNING: "+ (msg % args) + "\n")
00086 
00087     def error(self,msg,*args,**kwargs):
00088         self.f.write("ERROR: " + (msg % args) + "\n")
00089 
00090     info = critical
00091     debug = critical
00092 
00093 # Null logger is used when no output is generated. Does nothing.
00094 class NullLogger(object):
00095     def __getattribute__(self,name):
00096         return self
00097     def __call__(self,*args,**kwargs):
00098         return self
00099 
00100 # -----------------------------------------------------------------------------
00101 #                        === Lexing Engine ===
00102 #
00103 # The following Lexer class implements the lexer runtime.   There are only
00104 # a few public methods and attributes:
00105 #
00106 #    input()          -  Store a new string in the lexer
00107 #    token()          -  Get the next token
00108 #    clone()          -  Clone the lexer
00109 #
00110 #    lineno           -  Current line number
00111 #    lexpos           -  Current position in the input string
00112 # -----------------------------------------------------------------------------
00113 
00114 class Lexer:
00115     def __init__(self):
00116         self.lexre = None             # Master regular expression. This is a list of
00117                                       # tuples (re,findex) where re is a compiled
00118                                       # regular expression and findex is a list
00119                                       # mapping regex group numbers to rules
00120         self.lexretext = None         # Current regular expression strings
00121         self.lexstatere = {}          # Dictionary mapping lexer states to master regexs
00122         self.lexstateretext = {}      # Dictionary mapping lexer states to regex strings
00123         self.lexstaterenames = {}     # Dictionary mapping lexer states to symbol names
00124         self.lexstate = "INITIAL"     # Current lexer state
00125         self.lexstatestack = []       # Stack of lexer states
00126         self.lexstateinfo = None      # State information
00127         self.lexstateignore = {}      # Dictionary of ignored characters for each state
00128         self.lexstateerrorf = {}      # Dictionary of error functions for each state
00129         self.lexreflags = 0           # Optional re compile flags
00130         self.lexdata = None           # Actual input data (as a string)
00131         self.lexpos = 0               # Current position in input text
00132         self.lexlen = 0               # Length of the input text
00133         self.lexerrorf = None         # Error rule (if any)
00134         self.lextokens = None         # List of valid tokens
00135         self.lexignore = ""           # Ignored characters
00136         self.lexliterals = ""         # Literal characters that can be passed through
00137         self.lexmodule = None         # Module
00138         self.lineno = 1               # Current line number
00139         self.lexoptimize = 0          # Optimized mode
00140 
00141     def clone(self,object=None):
00142         c = copy.copy(self)
00143 
00144         # If the object parameter has been supplied, it means we are attaching the
00145         # lexer to a new object.  In this case, we have to rebind all methods in
00146         # the lexstatere and lexstateerrorf tables.
00147 
00148         if object:
00149             newtab = { }
00150             for key, ritem in self.lexstatere.items():
00151                 newre = []
00152                 for cre, findex in ritem:
00153                      newfindex = []
00154                      for f in findex:
00155                          if not f or not f[0]:
00156                              newfindex.append(f)
00157                              continue
00158                          newfindex.append((getattr(object,f[0].__name__),f[1]))
00159                 newre.append((cre,newfindex))
00160                 newtab[key] = newre
00161             c.lexstatere = newtab
00162             c.lexstateerrorf = { }
00163             for key, ef in self.lexstateerrorf.items():
00164                 c.lexstateerrorf[key] = getattr(object,ef.__name__)
00165             c.lexmodule = object
00166         return c
00167 
00168     # ------------------------------------------------------------
00169     # writetab() - Write lexer information to a table file
00170     # ------------------------------------------------------------
00171     def writetab(self,tabfile,outputdir=""):
00172         if isinstance(tabfile,types.ModuleType):
00173             return
00174         basetabfilename = tabfile.split(".")[-1]
00175         filename = os.path.join(outputdir,basetabfilename)+".py"
00176         tf = open(filename,"w")
00177         tf.write("# %s.py. This file automatically created by PLY (version %s). Don't edit!\n" % (tabfile,__version__))
00178         tf.write("_tabversion   = %s\n" % repr(__version__))
00179         tf.write("_lextokens    = %s\n" % repr(self.lextokens))
00180         tf.write("_lexreflags   = %s\n" % repr(self.lexreflags))
00181         tf.write("_lexliterals  = %s\n" % repr(self.lexliterals))
00182         tf.write("_lexstateinfo = %s\n" % repr(self.lexstateinfo))
00183 
00184         tabre = { }
00185         # Collect all functions in the initial state
00186         initial = self.lexstatere["INITIAL"]
00187         initialfuncs = []
00188         for part in initial:
00189             for f in part[1]:
00190                 if f and f[0]:
00191                     initialfuncs.append(f)
00192 
00193         for key, lre in self.lexstatere.items():
00194              titem = []
00195              for i in range(len(lre)):
00196                   titem.append((self.lexstateretext[key][i],_funcs_to_names(lre[i][1],self.lexstaterenames[key][i])))
00197              tabre[key] = titem
00198 
00199         tf.write("_lexstatere   = %s\n" % repr(tabre))
00200         tf.write("_lexstateignore = %s\n" % repr(self.lexstateignore))
00201 
00202         taberr = { }
00203         for key, ef in self.lexstateerrorf.items():
00204              if ef:
00205                   taberr[key] = ef.__name__
00206              else:
00207                   taberr[key] = None
00208         tf.write("_lexstateerrorf = %s\n" % repr(taberr))
00209         tf.close()
00210 
00211     # ------------------------------------------------------------
00212     # readtab() - Read lexer information from a tab file
00213     # ------------------------------------------------------------
00214     def readtab(self,tabfile,fdict):
00215         if isinstance(tabfile,types.ModuleType):
00216             lextab = tabfile
00217         else:
00218             if sys.version_info[0] < 3:
00219                 exec("import %s as lextab" % tabfile)
00220             else:
00221                 env = { }
00222                 exec("import %s as lextab" % tabfile, env,env)
00223                 lextab = env['lextab']
00224 
00225         if getattr(lextab,"_tabversion","0.0") != __version__:
00226             raise ImportError("Inconsistent PLY version")
00227 
00228         self.lextokens      = lextab._lextokens
00229         self.lexreflags     = lextab._lexreflags
00230         self.lexliterals    = lextab._lexliterals
00231         self.lexstateinfo   = lextab._lexstateinfo
00232         self.lexstateignore = lextab._lexstateignore
00233         self.lexstatere     = { }
00234         self.lexstateretext = { }
00235         for key,lre in lextab._lexstatere.items():
00236              titem = []
00237              txtitem = []
00238              for i in range(len(lre)):
00239                   titem.append((re.compile(lre[i][0],lextab._lexreflags | re.VERBOSE),_names_to_funcs(lre[i][1],fdict)))
00240                   txtitem.append(lre[i][0])
00241              self.lexstatere[key] = titem
00242              self.lexstateretext[key] = txtitem
00243         self.lexstateerrorf = { }
00244         for key,ef in lextab._lexstateerrorf.items():
00245              self.lexstateerrorf[key] = fdict[ef]
00246         self.begin('INITIAL')
00247 
00248     # ------------------------------------------------------------
00249     # input() - Push a new string into the lexer
00250     # ------------------------------------------------------------
00251     def input(self,s):
00252         # Pull off the first character to see if s looks like a string
00253         c = s[:1]
00254         if not isinstance(c,StringTypes):
00255             raise ValueError("Expected a string")
00256         self.lexdata = s
00257         self.lexpos = 0
00258         self.lexlen = len(s)
00259 
00260     # ------------------------------------------------------------
00261     # begin() - Changes the lexing state
00262     # ------------------------------------------------------------
00263     def begin(self,state):
00264         if not state in self.lexstatere:
00265             raise ValueError("Undefined state")
00266         self.lexre = self.lexstatere[state]
00267         self.lexretext = self.lexstateretext[state]
00268         self.lexignore = self.lexstateignore.get(state,"")
00269         self.lexerrorf = self.lexstateerrorf.get(state,None)
00270         self.lexstate = state
00271 
00272     # ------------------------------------------------------------
00273     # push_state() - Changes the lexing state and saves old on stack
00274     # ------------------------------------------------------------
00275     def push_state(self,state):
00276         self.lexstatestack.append(self.lexstate)
00277         self.begin(state)
00278 
00279     # ------------------------------------------------------------
00280     # pop_state() - Restores the previous state
00281     # ------------------------------------------------------------
00282     def pop_state(self):
00283         self.begin(self.lexstatestack.pop())
00284 
00285     # ------------------------------------------------------------
00286     # current_state() - Returns the current lexing state
00287     # ------------------------------------------------------------
00288     def current_state(self):
00289         return self.lexstate
00290 
00291     # ------------------------------------------------------------
00292     # skip() - Skip ahead n characters
00293     # ------------------------------------------------------------
00294     def skip(self,n):
00295         self.lexpos += n
00296 
00297     # ------------------------------------------------------------
00298     # opttoken() - Return the next token from the Lexer
00299     #
00300     # Note: This function has been carefully implemented to be as fast
00301     # as possible.  Don't make changes unless you really know what
00302     # you are doing
00303     # ------------------------------------------------------------
00304     def token(self):
00305         # Make local copies of frequently referenced attributes
00306         lexpos    = self.lexpos
00307         lexlen    = self.lexlen
00308         lexignore = self.lexignore
00309         lexdata   = self.lexdata
00310 
00311         while lexpos < lexlen:
00312             # This code provides some short-circuit code for whitespace, tabs, and other ignored characters
00313             if lexdata[lexpos] in lexignore:
00314                 lexpos += 1
00315                 continue
00316 
00317             # Look for a regular expression match
00318             for lexre,lexindexfunc in self.lexre:
00319                 m = lexre.match(lexdata,lexpos)
00320                 if not m: continue
00321 
00322                 # Create a token for return
00323                 tok = LexToken()
00324                 tok.value = m.group()
00325                 tok.lineno = self.lineno
00326                 tok.lexpos = lexpos
00327 
00328                 i = m.lastindex
00329                 func,tok.type = lexindexfunc[i]
00330 
00331                 if not func:
00332                    # If no token type was set, it's an ignored token
00333                    if tok.type:
00334                       self.lexpos = m.end()
00335                       return tok
00336                    else:
00337                       lexpos = m.end()
00338                       break
00339 
00340                 lexpos = m.end()
00341 
00342                 # If token is processed by a function, call it
00343 
00344                 tok.lexer = self      # Set additional attributes useful in token rules
00345                 self.lexmatch = m
00346                 self.lexpos = lexpos
00347 
00348                 newtok = func(tok)
00349 
00350                 # Every function must return a token, if nothing, we just move to next token
00351                 if not newtok:
00352                     lexpos    = self.lexpos         # This is here in case user has updated lexpos.
00353                     lexignore = self.lexignore      # This is here in case there was a state change
00354                     break
00355 
00356                 # Verify type of the token.  If not in the token map, raise an error
00357                 if not self.lexoptimize:
00358                     if not newtok.type in self.lextokens:
00359                         raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % (
00360                             func_code(func).co_filename, func_code(func).co_firstlineno,
00361                             func.__name__, newtok.type),lexdata[lexpos:])
00362 
00363                 return newtok
00364             else:
00365                 # No match, see if in literals
00366                 if lexdata[lexpos] in self.lexliterals:
00367                     tok = LexToken()
00368                     tok.value = lexdata[lexpos]
00369                     tok.lineno = self.lineno
00370                     tok.type = tok.value
00371                     tok.lexpos = lexpos
00372                     self.lexpos = lexpos + 1
00373                     return tok
00374 
00375                 # No match. Call t_error() if defined.
00376                 if self.lexerrorf:
00377                     tok = LexToken()
00378                     tok.value = self.lexdata[lexpos:]
00379                     tok.lineno = self.lineno
00380                     tok.type = "error"
00381                     tok.lexer = self
00382                     tok.lexpos = lexpos
00383                     self.lexpos = lexpos
00384                     newtok = self.lexerrorf(tok)
00385                     if lexpos == self.lexpos:
00386                         # Error method didn't change text position at all. This is an error.
00387                         raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:])
00388                     lexpos = self.lexpos
00389                     if not newtok: continue
00390                     return newtok
00391 
00392                 self.lexpos = lexpos
00393                 raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos],lexpos), lexdata[lexpos:])
00394 
00395         self.lexpos = lexpos + 1
00396         if self.lexdata is None:
00397              raise RuntimeError("No input string given with input()")
00398         return None
00399 
00400     # Iterator interface
00401     def __iter__(self):
00402         return self
00403 
00404     def next(self):
00405         t = self.token()
00406         if t is None:
00407             raise StopIteration
00408         return t
00409 
00410     __next__ = next
00411 
00412 # -----------------------------------------------------------------------------
00413 #                           ==== Lex Builder ===
00414 #
00415 # The functions and classes below are used to collect lexing information
00416 # and build a Lexer object from it.
00417 # -----------------------------------------------------------------------------
00418 
00419 # -----------------------------------------------------------------------------
00420 # get_caller_module_dict()
00421 #
00422 # This function returns a dictionary containing all of the symbols defined within
00423 # a caller further down the call stack.  This is used to get the environment
00424 # associated with the yacc() call if none was provided.
00425 # -----------------------------------------------------------------------------
00426 
00427 def get_caller_module_dict(levels):
00428     try:
00429         raise RuntimeError
00430     except RuntimeError:
00431         e,b,t = sys.exc_info()
00432         f = t.tb_frame
00433         while levels > 0:
00434             f = f.f_back                   
00435             levels -= 1
00436         ldict = f.f_globals.copy()
00437         if f.f_globals != f.f_locals:
00438             ldict.update(f.f_locals)
00439 
00440         return ldict
00441 
00442 # -----------------------------------------------------------------------------
00443 # _funcs_to_names()
00444 #
00445 # Given a list of regular expression functions, this converts it to a list
00446 # suitable for output to a table file
00447 # -----------------------------------------------------------------------------
00448 
00449 def _funcs_to_names(funclist,namelist):
00450     result = []
00451     for f,name in zip(funclist,namelist):
00452          if f and f[0]:
00453              result.append((name, f[1]))
00454          else:
00455              result.append(f)
00456     return result
00457 
00458 # -----------------------------------------------------------------------------
00459 # _names_to_funcs()
00460 #
00461 # Given a list of regular expression function names, this converts it back to
00462 # functions.
00463 # -----------------------------------------------------------------------------
00464 
00465 def _names_to_funcs(namelist,fdict):
00466      result = []
00467      for n in namelist:
00468           if n and n[0]:
00469               result.append((fdict[n[0]],n[1]))
00470           else:
00471               result.append(n)
00472      return result
00473 
00474 # -----------------------------------------------------------------------------
00475 # _form_master_re()
00476 #
00477 # This function takes a list of all of the regex components and attempts to
00478 # form the master regular expression.  Given limitations in the Python re
00479 # module, it may be necessary to break the master regex into separate expressions.
00480 # -----------------------------------------------------------------------------
00481 
00482 def _form_master_re(relist,reflags,ldict,toknames):
00483     if not relist: return []
00484     regex = "|".join(relist)
00485     try:
00486         lexre = re.compile(regex,re.VERBOSE | reflags)
00487 
00488         # Build the index to function map for the matching engine
00489         lexindexfunc = [ None ] * (max(lexre.groupindex.values())+1)
00490         lexindexnames = lexindexfunc[:]
00491 
00492         for f,i in lexre.groupindex.items():
00493             handle = ldict.get(f,None)
00494             if type(handle) in (types.FunctionType, types.MethodType):
00495                 lexindexfunc[i] = (handle,toknames[f])
00496                 lexindexnames[i] = f
00497             elif handle is not None:
00498                 lexindexnames[i] = f
00499                 if f.find("ignore_") > 0:
00500                     lexindexfunc[i] = (None,None)
00501                 else:
00502                     lexindexfunc[i] = (None, toknames[f])
00503         
00504         return [(lexre,lexindexfunc)],[regex],[lexindexnames]
00505     except Exception:
00506         m = int(len(relist)/2)
00507         if m == 0: m = 1
00508         llist, lre, lnames = _form_master_re(relist[:m],reflags,ldict,toknames)
00509         rlist, rre, rnames = _form_master_re(relist[m:],reflags,ldict,toknames)
00510         return llist+rlist, lre+rre, lnames+rnames
00511 
00512 # -----------------------------------------------------------------------------
00513 # def _statetoken(s,names)
00514 #
00515 # Given a declaration name s of the form "t_" and a dictionary whose keys are
00516 # state names, this function returns a tuple (states,tokenname) where states
00517 # is a tuple of state names and tokenname is the name of the token.  For example,
00518 # calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM')
00519 # -----------------------------------------------------------------------------
00520 
00521 def _statetoken(s,names):
00522     nonstate = 1
00523     parts = s.split("_")
00524     for i in range(1,len(parts)):
00525          if not parts[i] in names and parts[i] != 'ANY': break
00526     if i > 1:
00527        states = tuple(parts[1:i])
00528     else:
00529        states = ('INITIAL',)
00530 
00531     if 'ANY' in states:
00532        states = tuple(names)
00533 
00534     tokenname = "_".join(parts[i:])
00535     return (states,tokenname)
00536 
00537 
00538 # -----------------------------------------------------------------------------
00539 # LexerReflect()
00540 #
00541 # This class represents information needed to build a lexer as extracted from a
00542 # user's input file.
00543 # -----------------------------------------------------------------------------
00544 class LexerReflect(object):
00545     def __init__(self,ldict,log=None,reflags=0):
00546         self.ldict      = ldict
00547         self.error_func = None
00548         self.tokens     = []
00549         self.reflags    = reflags
00550         self.stateinfo  = { 'INITIAL' : 'inclusive'}
00551         self.files      = {}
00552         self.error      = 0
00553 
00554         if log is None:
00555             self.log = PlyLogger(sys.stderr)
00556         else:
00557             self.log = log
00558 
00559     # Get all of the basic information
00560     def get_all(self):
00561         self.get_tokens()
00562         self.get_literals()
00563         self.get_states()
00564         self.get_rules()
00565         
00566     # Validate all of the information
00567     def validate_all(self):
00568         self.validate_tokens()
00569         self.validate_literals()
00570         self.validate_rules()
00571         return self.error
00572 
00573     # Get the tokens map
00574     def get_tokens(self):
00575         tokens = self.ldict.get("tokens",None)
00576         if not tokens:
00577             self.log.error("No token list is defined")
00578             self.error = 1
00579             return
00580 
00581         if not isinstance(tokens,(list, tuple)):
00582             self.log.error("tokens must be a list or tuple")
00583             self.error = 1
00584             return
00585         
00586         if not tokens:
00587             self.log.error("tokens is empty")
00588             self.error = 1
00589             return
00590 
00591         self.tokens = tokens
00592 
00593     # Validate the tokens
00594     def validate_tokens(self):
00595         terminals = {}
00596         for n in self.tokens:
00597             if not _is_identifier.match(n):
00598                 self.log.error("Bad token name '%s'",n)
00599                 self.error = 1
00600             if n in terminals:
00601                 self.log.warning("Token '%s' multiply defined", n)
00602             terminals[n] = 1
00603 
00604     # Get the literals specifier
00605     def get_literals(self):
00606         self.literals = self.ldict.get("literals","")
00607 
00608     # Validate literals
00609     def validate_literals(self):
00610         try:
00611             for c in self.literals:
00612                 if not isinstance(c,StringTypes) or len(c) > 1:
00613                     self.log.error("Invalid literal %s. Must be a single character", repr(c))
00614                     self.error = 1
00615                     continue
00616 
00617         except TypeError:
00618             self.log.error("Invalid literals specification. literals must be a sequence of characters")
00619             self.error = 1
00620 
00621     def get_states(self):
00622         self.states = self.ldict.get("states",None)
00623         # Build statemap
00624         if self.states:
00625              if not isinstance(self.states,(tuple,list)):
00626                   self.log.error("states must be defined as a tuple or list")
00627                   self.error = 1
00628              else:
00629                   for s in self.states:
00630                         if not isinstance(s,tuple) or len(s) != 2:
00631                                self.log.error("Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')",repr(s))
00632                                self.error = 1
00633                                continue
00634                         name, statetype = s
00635                         if not isinstance(name,StringTypes):
00636                                self.log.error("State name %s must be a string", repr(name))
00637                                self.error = 1
00638                                continue
00639                         if not (statetype == 'inclusive' or statetype == 'exclusive'):
00640                                self.log.error("State type for state %s must be 'inclusive' or 'exclusive'",name)
00641                                self.error = 1
00642                                continue
00643                         if name in self.stateinfo:
00644                                self.log.error("State '%s' already defined",name)
00645                                self.error = 1
00646                                continue
00647                         self.stateinfo[name] = statetype
00648 
00649     # Get all of the symbols with a t_ prefix and sort them into various
00650     # categories (functions, strings, error functions, and ignore characters)
00651 
00652     def get_rules(self):
00653         tsymbols = [f for f in self.ldict if f[:2] == 't_' ]
00654 
00655         # Now build up a list of functions and a list of strings
00656 
00657         self.toknames = { }        # Mapping of symbols to token names
00658         self.funcsym =  { }        # Symbols defined as functions
00659         self.strsym =   { }        # Symbols defined as strings
00660         self.ignore   = { }        # Ignore strings by state
00661         self.errorf   = { }        # Error functions by state
00662 
00663         for s in self.stateinfo:
00664              self.funcsym[s] = []
00665              self.strsym[s] = []
00666 
00667         if len(tsymbols) == 0:
00668             self.log.error("No rules of the form t_rulename are defined")
00669             self.error = 1
00670             return
00671 
00672         for f in tsymbols:
00673             t = self.ldict[f]
00674             states, tokname = _statetoken(f,self.stateinfo)
00675             self.toknames[f] = tokname
00676 
00677             if hasattr(t,"__call__"):
00678                 if tokname == 'error':
00679                     for s in states:
00680                         self.errorf[s] = t
00681                 elif tokname == 'ignore':
00682                     line = func_code(t).co_firstlineno
00683                     file = func_code(t).co_filename
00684                     self.log.error("%s:%d: Rule '%s' must be defined as a string",file,line,t.__name__)
00685                     self.error = 1
00686                 else:
00687                     for s in states: 
00688                         self.funcsym[s].append((f,t))
00689             elif isinstance(t, StringTypes):
00690                 if tokname == 'ignore':
00691                     for s in states:
00692                         self.ignore[s] = t
00693                     if "\\" in t:
00694                         self.log.warning("%s contains a literal backslash '\\'",f)
00695 
00696                 elif tokname == 'error':
00697                     self.log.error("Rule '%s' must be defined as a function", f)
00698                     self.error = 1
00699                 else:
00700                     for s in states: 
00701                         self.strsym[s].append((f,t))
00702             else:
00703                 self.log.error("%s not defined as a function or string", f)
00704                 self.error = 1
00705 
00706         # Sort the functions by line number
00707         for f in self.funcsym.values():
00708             if sys.version_info[0] < 3:
00709                 f.sort(lambda x,y: cmp(func_code(x[1]).co_firstlineno,func_code(y[1]).co_firstlineno))
00710             else:
00711                 # Python 3.0
00712                 f.sort(key=lambda x: func_code(x[1]).co_firstlineno)
00713 
00714         # Sort the strings by regular expression length
00715         for s in self.strsym.values():
00716             if sys.version_info[0] < 3:
00717                 s.sort(lambda x,y: (len(x[1]) < len(y[1])) - (len(x[1]) > len(y[1])))
00718             else:
00719                 # Python 3.0
00720                 s.sort(key=lambda x: len(x[1]),reverse=True)
00721 
00722     # Validate all of the t_rules collected 
00723     def validate_rules(self):
00724         for state in self.stateinfo:
00725             # Validate all rules defined by functions
00726 
00727             
00728 
00729             for fname, f in self.funcsym[state]:
00730                 line = func_code(f).co_firstlineno
00731                 file = func_code(f).co_filename
00732                 self.files[file] = 1
00733 
00734                 tokname = self.toknames[fname]
00735                 if isinstance(f, types.MethodType):
00736                     reqargs = 2
00737                 else:
00738                     reqargs = 1
00739                 nargs = func_code(f).co_argcount
00740                 if nargs > reqargs:
00741                     self.log.error("%s:%d: Rule '%s' has too many arguments",file,line,f.__name__)
00742                     self.error = 1
00743                     continue
00744 
00745                 if nargs < reqargs:
00746                     self.log.error("%s:%d: Rule '%s' requires an argument", file,line,f.__name__)
00747                     self.error = 1
00748                     continue
00749 
00750                 if not f.__doc__:
00751                     self.log.error("%s:%d: No regular expression defined for rule '%s'",file,line,f.__name__)
00752                     self.error = 1
00753                     continue
00754 
00755                 try:
00756                     c = re.compile("(?P<%s>%s)" % (fname,f.__doc__), re.VERBOSE | self.reflags)
00757                     if c.match(""):
00758                         self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file,line,f.__name__)
00759                         self.error = 1
00760                 except re.error:
00761                     _etype, e, _etrace = sys.exc_info()
00762                     self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file,line,f.__name__,e)
00763                     if '#' in f.__doc__:
00764                         self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'",file,line, f.__name__)
00765                     self.error = 1
00766 
00767             # Validate all rules defined by strings
00768             for name,r in self.strsym[state]:
00769                 tokname = self.toknames[name]
00770                 if tokname == 'error':
00771                     self.log.error("Rule '%s' must be defined as a function", name)
00772                     self.error = 1
00773                     continue
00774 
00775                 if not tokname in self.tokens and tokname.find("ignore_") < 0:
00776                     self.log.error("Rule '%s' defined for an unspecified token %s",name,tokname)
00777                     self.error = 1
00778                     continue
00779 
00780                 try:
00781                     c = re.compile("(?P<%s>%s)" % (name,r),re.VERBOSE | self.reflags)
00782                     if (c.match("")):
00783                          self.log.error("Regular expression for rule '%s' matches empty string",name)
00784                          self.error = 1
00785                 except re.error:
00786                     _etype, e, _etrace = sys.exc_info()
00787                     self.log.error("Invalid regular expression for rule '%s'. %s",name,e)
00788                     if '#' in r:
00789                          self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'",name)
00790                     self.error = 1
00791 
00792             if not self.funcsym[state] and not self.strsym[state]:
00793                 self.log.error("No rules defined for state '%s'",state)
00794                 self.error = 1
00795 
00796             # Validate the error function
00797             efunc = self.errorf.get(state,None)
00798             if efunc:
00799                 f = efunc
00800                 line = func_code(f).co_firstlineno
00801                 file = func_code(f).co_filename
00802                 self.files[file] = 1
00803 
00804                 if isinstance(f, types.MethodType):
00805                     reqargs = 2
00806                 else:
00807                     reqargs = 1
00808                 nargs = func_code(f).co_argcount
00809                 if nargs > reqargs:
00810                     self.log.error("%s:%d: Rule '%s' has too many arguments",file,line,f.__name__)
00811                     self.error = 1
00812 
00813                 if nargs < reqargs:
00814                     self.log.error("%s:%d: Rule '%s' requires an argument", file,line,f.__name__)
00815                     self.error = 1
00816 
00817         for f in self.files:
00818             self.validate_file(f)
00819 
00820 
00821     # -----------------------------------------------------------------------------
00822     # validate_file()
00823     #
00824     # This checks to see if there are duplicated t_rulename() functions or strings
00825     # in the parser input file.  This is done using a simple regular expression
00826     # match on each line in the given file.  
00827     # -----------------------------------------------------------------------------
00828 
00829     def validate_file(self,filename):
00830         import os.path
00831         base,ext = os.path.splitext(filename)
00832         if ext != '.py': return         # No idea what the file is. Return OK
00833 
00834         try:
00835             f = open(filename)
00836             lines = f.readlines()
00837             f.close()
00838         except IOError:
00839             return                      # Couldn't find the file.  Don't worry about it
00840 
00841         fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(')
00842         sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=')
00843 
00844         counthash = { }
00845         linen = 1
00846         for l in lines:
00847             m = fre.match(l)
00848             if not m:
00849                 m = sre.match(l)
00850             if m:
00851                 name = m.group(1)
00852                 prev = counthash.get(name)
00853                 if not prev:
00854                     counthash[name] = linen
00855                 else:
00856                     self.log.error("%s:%d: Rule %s redefined. Previously defined on line %d",filename,linen,name,prev)
00857                     self.error = 1
00858             linen += 1
00859             
00860 # -----------------------------------------------------------------------------
00861 # lex(module)
00862 #
00863 # Build all of the regular expression rules from definitions in the supplied module
00864 # -----------------------------------------------------------------------------
00865 def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,nowarn=0,outputdir="", debuglog=None, errorlog=None):
00866     global lexer
00867     ldict = None
00868     stateinfo  = { 'INITIAL' : 'inclusive'}
00869     lexobj = Lexer()
00870     lexobj.lexoptimize = optimize
00871     global token,input
00872 
00873     if errorlog is None:
00874         errorlog = PlyLogger(sys.stderr)
00875 
00876     if debug:
00877         if debuglog is None:
00878             debuglog = PlyLogger(sys.stderr)
00879 
00880     # Get the module dictionary used for the lexer
00881     if object: module = object
00882 
00883     if module:
00884         _items = [(k,getattr(module,k)) for k in dir(module)]
00885         ldict = dict(_items)
00886     else:
00887         ldict = get_caller_module_dict(2)
00888 
00889     # Collect parser information from the dictionary
00890     linfo = LexerReflect(ldict,log=errorlog,reflags=reflags)
00891     linfo.get_all()
00892     if not optimize:
00893         if linfo.validate_all():
00894             raise SyntaxError("Can't build lexer")
00895 
00896     if optimize and lextab:
00897         try:
00898             lexobj.readtab(lextab,ldict)
00899             token = lexobj.token
00900             input = lexobj.input
00901             lexer = lexobj
00902             return lexobj
00903 
00904         except ImportError:
00905             pass
00906 
00907     # Dump some basic debugging information
00908     if debug:
00909         debuglog.info("lex: tokens   = %r", linfo.tokens)
00910         debuglog.info("lex: literals = %r", linfo.literals)
00911         debuglog.info("lex: states   = %r", linfo.stateinfo)
00912 
00913     # Build a dictionary of valid token names
00914     lexobj.lextokens = { }
00915     for n in linfo.tokens:
00916         lexobj.lextokens[n] = 1
00917 
00918     # Get literals specification
00919     if isinstance(linfo.literals,(list,tuple)):
00920         lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals)
00921     else:
00922         lexobj.lexliterals = linfo.literals
00923 
00924     # Get the stateinfo dictionary
00925     stateinfo = linfo.stateinfo
00926 
00927     regexs = { }
00928     # Build the master regular expressions
00929     for state in stateinfo:
00930         regex_list = []
00931 
00932         # Add rules defined by functions first
00933         for fname, f in linfo.funcsym[state]:
00934             line = func_code(f).co_firstlineno
00935             file = func_code(f).co_filename
00936             regex_list.append("(?P<%s>%s)" % (fname,f.__doc__))
00937             if debug:
00938                 debuglog.info("lex: Adding rule %s -> '%s' (state '%s')",fname,f.__doc__, state)
00939 
00940         # Now add all of the simple rules
00941         for name,r in linfo.strsym[state]:
00942             regex_list.append("(?P<%s>%s)" % (name,r))
00943             if debug:
00944                 debuglog.info("lex: Adding rule %s -> '%s' (state '%s')",name,r, state)
00945 
00946         regexs[state] = regex_list
00947 
00948     # Build the master regular expressions
00949 
00950     if debug:
00951         debuglog.info("lex: ==== MASTER REGEXS FOLLOW ====")
00952 
00953     for state in regexs:
00954         lexre, re_text, re_names = _form_master_re(regexs[state],reflags,ldict,linfo.toknames)
00955         lexobj.lexstatere[state] = lexre
00956         lexobj.lexstateretext[state] = re_text
00957         lexobj.lexstaterenames[state] = re_names
00958         if debug:
00959             for i in range(len(re_text)):
00960                 debuglog.info("lex: state '%s' : regex[%d] = '%s'",state, i, re_text[i])
00961 
00962     # For inclusive states, we need to add the regular expressions from the INITIAL state
00963     for state,stype in stateinfo.items():
00964         if state != "INITIAL" and stype == 'inclusive':
00965              lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL'])
00966              lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL'])
00967              lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL'])
00968 
00969     lexobj.lexstateinfo = stateinfo
00970     lexobj.lexre = lexobj.lexstatere["INITIAL"]
00971     lexobj.lexretext = lexobj.lexstateretext["INITIAL"]
00972     lexobj.lexreflags = reflags
00973 
00974     # Set up ignore variables
00975     lexobj.lexstateignore = linfo.ignore
00976     lexobj.lexignore = lexobj.lexstateignore.get("INITIAL","")
00977 
00978     # Set up error functions
00979     lexobj.lexstateerrorf = linfo.errorf
00980     lexobj.lexerrorf = linfo.errorf.get("INITIAL",None)
00981     if not lexobj.lexerrorf:
00982         errorlog.warning("No t_error rule is defined")
00983 
00984     # Check state information for ignore and error rules
00985     for s,stype in stateinfo.items():
00986         if stype == 'exclusive':
00987               if not s in linfo.errorf:
00988                    errorlog.warning("No error rule is defined for exclusive state '%s'", s)
00989               if not s in linfo.ignore and lexobj.lexignore:
00990                    errorlog.warning("No ignore rule is defined for exclusive state '%s'", s)
00991         elif stype == 'inclusive':
00992               if not s in linfo.errorf:
00993                    linfo.errorf[s] = linfo.errorf.get("INITIAL",None)
00994               if not s in linfo.ignore:
00995                    linfo.ignore[s] = linfo.ignore.get("INITIAL","")
00996 
00997     # Create global versions of the token() and input() functions
00998     token = lexobj.token
00999     input = lexobj.input
01000     lexer = lexobj
01001 
01002     # If in optimize mode, we write the lextab
01003     if lextab and optimize:
01004         lexobj.writetab(lextab,outputdir)
01005 
01006     return lexobj
01007 
01008 # -----------------------------------------------------------------------------
01009 # runmain()
01010 #
01011 # This runs the lexer as a main program
01012 # -----------------------------------------------------------------------------
01013 
01014 def runmain(lexer=None,data=None):
01015     if not data:
01016         try:
01017             filename = sys.argv[1]
01018             f = open(filename)
01019             data = f.read()
01020             f.close()
01021         except IndexError:
01022             sys.stdout.write("Reading from standard input (type EOF to end):\n")
01023             data = sys.stdin.read()
01024 
01025     if lexer:
01026         _input = lexer.input
01027     else:
01028         _input = input
01029     _input(data)
01030     if lexer:
01031         _token = lexer.token
01032     else:
01033         _token = token
01034 
01035     while 1:
01036         tok = _token()
01037         if not tok: break
01038         sys.stdout.write("(%s,%r,%d,%d)\n" % (tok.type, tok.value, tok.lineno,tok.lexpos))
01039 
01040 # -----------------------------------------------------------------------------
01041 # @TOKEN(regex)
01042 #
01043 # This decorator function can be used to set the regex expression on a function
01044 # when its docstring might need to be set in an alternative way
01045 # -----------------------------------------------------------------------------
01046 
01047 def TOKEN(r):
01048     def set_doc(f):
01049         if hasattr(r,"__call__"):
01050             f.__doc__ = r.__doc__
01051         else:
01052             f.__doc__ = r
01053         return f
01054     return set_doc
01055 
01056 # Alternative spelling of the TOKEN decorator
01057 Token = TOKEN
01058