Back to index

enigmail  1.4.3
Public Member Functions | Public Attributes | Static Private Attributes
ply.lex.Lexer Class Reference

List of all members.

Public Member Functions

def __init__
def clone
def writetab
def readtab
def input
def begin
def push_state
def pop_state
def current_state
def skip
def token
def __iter__
def next

Public Attributes

 lexre
 lexretext
 lexstatere
 lexstateretext
 lexstaterenames
 lexstate
 lexstatestack
 lexstateinfo
 lexstateignore
 lexstateerrorf
 lexreflags
 lexdata
 lexpos
 lexlen
 lexerrorf
 lextokens
 lexignore
 lexliterals
 lexmodule
 lineno
 lexoptimize
 lexmatch

Static Private Attributes

 __next__ = next

Detailed Description

Definition at line 114 of file lex.py.


Constructor & Destructor Documentation

def ply.lex.Lexer.__init__ (   self)

Definition at line 115 of file lex.py.

00115 
00116     def __init__(self):
00117         self.lexre = None             # Master regular expression. This is a list of
00118                                       # tuples (re,findex) where re is a compiled
00119                                       # regular expression and findex is a list
00120                                       # mapping regex group numbers to rules
00121         self.lexretext = None         # Current regular expression strings
00122         self.lexstatere = {}          # Dictionary mapping lexer states to master regexs
00123         self.lexstateretext = {}      # Dictionary mapping lexer states to regex strings
00124         self.lexstaterenames = {}     # Dictionary mapping lexer states to symbol names
00125         self.lexstate = "INITIAL"     # Current lexer state
00126         self.lexstatestack = []       # Stack of lexer states
00127         self.lexstateinfo = None      # State information
00128         self.lexstateignore = {}      # Dictionary of ignored characters for each state
00129         self.lexstateerrorf = {}      # Dictionary of error functions for each state
00130         self.lexreflags = 0           # Optional re compile flags
00131         self.lexdata = None           # Actual input data (as a string)
00132         self.lexpos = 0               # Current position in input text
00133         self.lexlen = 0               # Length of the input text
00134         self.lexerrorf = None         # Error rule (if any)
00135         self.lextokens = None         # List of valid tokens
00136         self.lexignore = ""           # Ignored characters
00137         self.lexliterals = ""         # Literal characters that can be passed through
00138         self.lexmodule = None         # Module
00139         self.lineno = 1               # Current line number
00140         self.lexoptimize = 0          # Optimized mode


Member Function Documentation

def ply.lex.Lexer.__iter__ (   self)

Definition at line 401 of file lex.py.

00401 
00402     def __iter__(self):
00403         return self

def ply.lex.Lexer.begin (   self,
  state 
)

Definition at line 263 of file lex.py.

00263 
00264     def begin(self,state):
00265         if not state in self.lexstatere:
00266             raise ValueError("Undefined state")
00267         self.lexre = self.lexstatere[state]
00268         self.lexretext = self.lexstateretext[state]
00269         self.lexignore = self.lexstateignore.get(state,"")
00270         self.lexerrorf = self.lexstateerrorf.get(state,None)
00271         self.lexstate = state

Here is the caller graph for this function:

def ply.lex.Lexer.clone (   self,
  object = None 
)

Definition at line 141 of file lex.py.

00141 
00142     def clone(self,object=None):
00143         c = copy.copy(self)
00144 
00145         # If the object parameter has been supplied, it means we are attaching the
00146         # lexer to a new object.  In this case, we have to rebind all methods in
00147         # the lexstatere and lexstateerrorf tables.
00148 
00149         if object:
00150             newtab = { }
00151             for key, ritem in self.lexstatere.items():
00152                 newre = []
00153                 for cre, findex in ritem:
00154                      newfindex = []
00155                      for f in findex:
00156                          if not f or not f[0]:
00157                              newfindex.append(f)
00158                              continue
00159                          newfindex.append((getattr(object,f[0].__name__),f[1]))
00160                 newre.append((cre,newfindex))
00161                 newtab[key] = newre
00162             c.lexstatere = newtab
00163             c.lexstateerrorf = { }
00164             for key, ef in self.lexstateerrorf.items():
00165                 c.lexstateerrorf[key] = getattr(object,ef.__name__)
00166             c.lexmodule = object
00167         return c

Definition at line 288 of file lex.py.

00288 
00289     def current_state(self):
00290         return self.lexstate

def ply.lex.Lexer.input (   self,
  s 
)

Definition at line 251 of file lex.py.

00251 
00252     def input(self,s):
00253         # Pull off the first character to see if s looks like a string
00254         c = s[:1]
00255         if not isinstance(c,StringTypes):
00256             raise ValueError("Expected a string")
00257         self.lexdata = s
00258         self.lexpos = 0
00259         self.lexlen = len(s)

def ply.lex.Lexer.next (   self)

Definition at line 404 of file lex.py.

00404 
00405     def next(self):
00406         t = self.token()
00407         if t is None:
00408             raise StopIteration
00409         return t

Here is the call graph for this function:

def ply.lex.Lexer.pop_state (   self)

Definition at line 282 of file lex.py.

00282 
00283     def pop_state(self):
00284         self.begin(self.lexstatestack.pop())

Here is the call graph for this function:

def ply.lex.Lexer.push_state (   self,
  state 
)

Definition at line 275 of file lex.py.

00275 
00276     def push_state(self,state):
00277         self.lexstatestack.append(self.lexstate)
00278         self.begin(state)

Here is the call graph for this function:

def ply.lex.Lexer.readtab (   self,
  tabfile,
  fdict 
)

Definition at line 214 of file lex.py.

00214 
00215     def readtab(self,tabfile,fdict):
00216         if isinstance(tabfile,types.ModuleType):
00217             lextab = tabfile
00218         else:
00219             if sys.version_info[0] < 3:
00220                 exec("import %s as lextab" % tabfile)
00221             else:
00222                 env = { }
00223                 exec("import %s as lextab" % tabfile, env,env)
00224                 lextab = env['lextab']
00225 
00226         if getattr(lextab,"_tabversion","0.0") != __version__:
00227             raise ImportError("Inconsistent PLY version")
00228 
00229         self.lextokens      = lextab._lextokens
00230         self.lexreflags     = lextab._lexreflags
00231         self.lexliterals    = lextab._lexliterals
00232         self.lexstateinfo   = lextab._lexstateinfo
00233         self.lexstateignore = lextab._lexstateignore
00234         self.lexstatere     = { }
00235         self.lexstateretext = { }
00236         for key,lre in lextab._lexstatere.items():
00237              titem = []
00238              txtitem = []
00239              for i in range(len(lre)):
00240                   titem.append((re.compile(lre[i][0],lextab._lexreflags | re.VERBOSE),_names_to_funcs(lre[i][1],fdict)))
00241                   txtitem.append(lre[i][0])
00242              self.lexstatere[key] = titem
00243              self.lexstateretext[key] = txtitem
00244         self.lexstateerrorf = { }
00245         for key,ef in lextab._lexstateerrorf.items():
00246              self.lexstateerrorf[key] = fdict[ef]
00247         self.begin('INITIAL')

Here is the call graph for this function:

def ply.lex.Lexer.skip (   self,
  n 
)

Definition at line 294 of file lex.py.

00294 
00295     def skip(self,n):
00296         self.lexpos += n

def ply.lex.Lexer.token (   self)

Definition at line 304 of file lex.py.

00304 
00305     def token(self):
00306         # Make local copies of frequently referenced attributes
00307         lexpos    = self.lexpos
00308         lexlen    = self.lexlen
00309         lexignore = self.lexignore
00310         lexdata   = self.lexdata
00311 
00312         while lexpos < lexlen:
00313             # This code provides some short-circuit code for whitespace, tabs, and other ignored characters
00314             if lexdata[lexpos] in lexignore:
00315                 lexpos += 1
00316                 continue
00317 
00318             # Look for a regular expression match
00319             for lexre,lexindexfunc in self.lexre:
00320                 m = lexre.match(lexdata,lexpos)
00321                 if not m: continue
00322 
00323                 # Create a token for return
00324                 tok = LexToken()
00325                 tok.value = m.group()
00326                 tok.lineno = self.lineno
00327                 tok.lexpos = lexpos
00328 
00329                 i = m.lastindex
00330                 func,tok.type = lexindexfunc[i]
00331 
00332                 if not func:
00333                    # If no token type was set, it's an ignored token
00334                    if tok.type:
00335                       self.lexpos = m.end()
00336                       return tok
00337                    else:
00338                       lexpos = m.end()
00339                       break
00340 
00341                 lexpos = m.end()
00342 
00343                 # If token is processed by a function, call it
00344 
00345                 tok.lexer = self      # Set additional attributes useful in token rules
00346                 self.lexmatch = m
00347                 self.lexpos = lexpos
00348 
00349                 newtok = func(tok)
00350 
00351                 # Every function must return a token, if nothing, we just move to next token
00352                 if not newtok:
00353                     lexpos    = self.lexpos         # This is here in case user has updated lexpos.
00354                     lexignore = self.lexignore      # This is here in case there was a state change
00355                     break
00356 
00357                 # Verify type of the token.  If not in the token map, raise an error
00358                 if not self.lexoptimize:
00359                     if not newtok.type in self.lextokens:
00360                         raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % (
00361                             func_code(func).co_filename, func_code(func).co_firstlineno,
00362                             func.__name__, newtok.type),lexdata[lexpos:])
00363 
00364                 return newtok
00365             else:
00366                 # No match, see if in literals
00367                 if lexdata[lexpos] in self.lexliterals:
00368                     tok = LexToken()
00369                     tok.value = lexdata[lexpos]
00370                     tok.lineno = self.lineno
00371                     tok.type = tok.value
00372                     tok.lexpos = lexpos
00373                     self.lexpos = lexpos + 1
00374                     return tok
00375 
00376                 # No match. Call t_error() if defined.
00377                 if self.lexerrorf:
00378                     tok = LexToken()
00379                     tok.value = self.lexdata[lexpos:]
00380                     tok.lineno = self.lineno
00381                     tok.type = "error"
00382                     tok.lexer = self
00383                     tok.lexpos = lexpos
00384                     self.lexpos = lexpos
00385                     newtok = self.lexerrorf(tok)
00386                     if lexpos == self.lexpos:
00387                         # Error method didn't change text position at all. This is an error.
00388                         raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:])
00389                     lexpos = self.lexpos
00390                     if not newtok: continue
00391                     return newtok
00392 
00393                 self.lexpos = lexpos
00394                 raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos],lexpos), lexdata[lexpos:])
00395 
00396         self.lexpos = lexpos + 1
00397         if self.lexdata is None:
00398              raise RuntimeError("No input string given with input()")
00399         return None

Here is the caller graph for this function:

def ply.lex.Lexer.writetab (   self,
  tabfile,
  outputdir = "" 
)

Definition at line 171 of file lex.py.

00171 
00172     def writetab(self,tabfile,outputdir=""):
00173         if isinstance(tabfile,types.ModuleType):
00174             return
00175         basetabfilename = tabfile.split(".")[-1]
00176         filename = os.path.join(outputdir,basetabfilename)+".py"
00177         tf = open(filename,"w")
00178         tf.write("# %s.py. This file automatically created by PLY (version %s). Don't edit!\n" % (tabfile,__version__))
00179         tf.write("_tabversion   = %s\n" % repr(__version__))
00180         tf.write("_lextokens    = %s\n" % repr(self.lextokens))
00181         tf.write("_lexreflags   = %s\n" % repr(self.lexreflags))
00182         tf.write("_lexliterals  = %s\n" % repr(self.lexliterals))
00183         tf.write("_lexstateinfo = %s\n" % repr(self.lexstateinfo))
00184 
00185         tabre = { }
00186         # Collect all functions in the initial state
00187         initial = self.lexstatere["INITIAL"]
00188         initialfuncs = []
00189         for part in initial:
00190             for f in part[1]:
00191                 if f and f[0]:
00192                     initialfuncs.append(f)
00193 
00194         for key, lre in self.lexstatere.items():
00195              titem = []
00196              for i in range(len(lre)):
00197                   titem.append((self.lexstateretext[key][i],_funcs_to_names(lre[i][1],self.lexstaterenames[key][i])))
00198              tabre[key] = titem
00199 
00200         tf.write("_lexstatere   = %s\n" % repr(tabre))
00201         tf.write("_lexstateignore = %s\n" % repr(self.lexstateignore))
00202 
00203         taberr = { }
00204         for key, ef in self.lexstateerrorf.items():
00205              if ef:
00206                   taberr[key] = ef.__name__
00207              else:
00208                   taberr[key] = None
00209         tf.write("_lexstateerrorf = %s\n" % repr(taberr))
00210         tf.close()

Here is the call graph for this function:


Member Data Documentation

ply.lex.Lexer.__next__ = next [static, private]

Definition at line 410 of file lex.py.

Definition at line 130 of file lex.py.

Definition at line 133 of file lex.py.

Definition at line 135 of file lex.py.

Definition at line 132 of file lex.py.

Definition at line 136 of file lex.py.

Definition at line 345 of file lex.py.

Definition at line 137 of file lex.py.

Definition at line 139 of file lex.py.

Definition at line 131 of file lex.py.

Definition at line 116 of file lex.py.

Definition at line 129 of file lex.py.

Definition at line 120 of file lex.py.

Definition at line 124 of file lex.py.

Definition at line 128 of file lex.py.

Definition at line 127 of file lex.py.

Definition at line 126 of file lex.py.

Definition at line 121 of file lex.py.

Definition at line 123 of file lex.py.

Definition at line 122 of file lex.py.

Definition at line 125 of file lex.py.

Definition at line 134 of file lex.py.

Definition at line 138 of file lex.py.


The documentation for this class was generated from the following file: