Back to index

enigmail  1.4.3
Public Member Functions | Public Attributes
ply.lex.LexerReflect Class Reference

List of all members.

Public Member Functions

def __init__
def get_all
def validate_all
def get_tokens
def validate_tokens
def get_literals
def validate_literals
def get_states
def get_rules
def validate_rules
def validate_file

Public Attributes

 ldict
 error_func
 tokens
 reflags
 stateinfo
 files
 error
 log
 literals
 states
 toknames
 funcsym
 strsym
 ignore
 errorf

Detailed Description

Definition at line 544 of file lex.py.


Constructor & Destructor Documentation

def ply.lex.LexerReflect.__init__ (   self,
  ldict,
  log = None,
  reflags = 0 
)

Definition at line 545 of file lex.py.

00545 
00546     def __init__(self,ldict,log=None,reflags=0):
00547         self.ldict      = ldict
00548         self.error_func = None
00549         self.tokens     = []
00550         self.reflags    = reflags
00551         self.stateinfo  = { 'INITIAL' : 'inclusive'}
00552         self.files      = {}
00553         self.error      = 0
00554 
00555         if log is None:
00556             self.log = PlyLogger(sys.stderr)
00557         else:
00558             self.log = log


Member Function Documentation

Definition at line 560 of file lex.py.

00560 
00561     def get_all(self):
00562         self.get_tokens()
00563         self.get_literals()
00564         self.get_states()
00565         self.get_rules()
        

Here is the call graph for this function:

Definition at line 605 of file lex.py.

00605 
00606     def get_literals(self):
00607         self.literals = self.ldict.get("literals","")

Here is the caller graph for this function:

Definition at line 652 of file lex.py.

00652 
00653     def get_rules(self):
00654         tsymbols = [f for f in self.ldict if f[:2] == 't_' ]
00655 
00656         # Now build up a list of functions and a list of strings
00657 
00658         self.toknames = { }        # Mapping of symbols to token names
00659         self.funcsym =  { }        # Symbols defined as functions
00660         self.strsym =   { }        # Symbols defined as strings
00661         self.ignore   = { }        # Ignore strings by state
00662         self.errorf   = { }        # Error functions by state
00663 
00664         for s in self.stateinfo:
00665              self.funcsym[s] = []
00666              self.strsym[s] = []
00667 
00668         if len(tsymbols) == 0:
00669             self.log.error("No rules of the form t_rulename are defined")
00670             self.error = 1
00671             return
00672 
00673         for f in tsymbols:
00674             t = self.ldict[f]
00675             states, tokname = _statetoken(f,self.stateinfo)
00676             self.toknames[f] = tokname
00677 
00678             if hasattr(t,"__call__"):
00679                 if tokname == 'error':
00680                     for s in states:
00681                         self.errorf[s] = t
00682                 elif tokname == 'ignore':
00683                     line = func_code(t).co_firstlineno
00684                     file = func_code(t).co_filename
00685                     self.log.error("%s:%d: Rule '%s' must be defined as a string",file,line,t.__name__)
00686                     self.error = 1
00687                 else:
00688                     for s in states: 
00689                         self.funcsym[s].append((f,t))
00690             elif isinstance(t, StringTypes):
00691                 if tokname == 'ignore':
00692                     for s in states:
00693                         self.ignore[s] = t
00694                     if "\\" in t:
00695                         self.log.warning("%s contains a literal backslash '\\'",f)
00696 
00697                 elif tokname == 'error':
00698                     self.log.error("Rule '%s' must be defined as a function", f)
00699                     self.error = 1
00700                 else:
00701                     for s in states: 
00702                         self.strsym[s].append((f,t))
00703             else:
00704                 self.log.error("%s not defined as a function or string", f)
00705                 self.error = 1
00706 
00707         # Sort the functions by line number
00708         for f in self.funcsym.values():
00709             if sys.version_info[0] < 3:
00710                 f.sort(lambda x,y: cmp(func_code(x[1]).co_firstlineno,func_code(y[1]).co_firstlineno))
00711             else:
00712                 # Python 3.0
00713                 f.sort(key=lambda x: func_code(x[1]).co_firstlineno)
00714 
00715         # Sort the strings by regular expression length
00716         for s in self.strsym.values():
00717             if sys.version_info[0] < 3:
00718                 s.sort(lambda x,y: (len(x[1]) < len(y[1])) - (len(x[1]) > len(y[1])))
00719             else:
00720                 # Python 3.0
00721                 s.sort(key=lambda x: len(x[1]),reverse=True)

Here is the caller graph for this function:

Definition at line 621 of file lex.py.

00621 
00622     def get_states(self):
00623         self.states = self.ldict.get("states",None)
00624         # Build statemap
00625         if self.states:
00626              if not isinstance(self.states,(tuple,list)):
00627                   self.log.error("states must be defined as a tuple or list")
00628                   self.error = 1
00629              else:
00630                   for s in self.states:
00631                         if not isinstance(s,tuple) or len(s) != 2:
00632                                self.log.error("Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')",repr(s))
00633                                self.error = 1
00634                                continue
00635                         name, statetype = s
00636                         if not isinstance(name,StringTypes):
00637                                self.log.error("State name %s must be a string", repr(name))
00638                                self.error = 1
00639                                continue
00640                         if not (statetype == 'inclusive' or statetype == 'exclusive'):
00641                                self.log.error("State type for state %s must be 'inclusive' or 'exclusive'",name)
00642                                self.error = 1
00643                                continue
00644                         if name in self.stateinfo:
00645                                self.log.error("State '%s' already defined",name)
00646                                self.error = 1
00647                                continue
00648                         self.stateinfo[name] = statetype

Here is the caller graph for this function:

Definition at line 574 of file lex.py.

00574 
00575     def get_tokens(self):
00576         tokens = self.ldict.get("tokens",None)
00577         if not tokens:
00578             self.log.error("No token list is defined")
00579             self.error = 1
00580             return
00581 
00582         if not isinstance(tokens,(list, tuple)):
00583             self.log.error("tokens must be a list or tuple")
00584             self.error = 1
00585             return
00586         
00587         if not tokens:
00588             self.log.error("tokens is empty")
00589             self.error = 1
00590             return
00591 
00592         self.tokens = tokens

Here is the caller graph for this function:

Definition at line 567 of file lex.py.

00567 
00568     def validate_all(self):
00569         self.validate_tokens()
00570         self.validate_literals()
00571         self.validate_rules()
00572         return self.error

Here is the call graph for this function:

def ply.lex.LexerReflect.validate_file (   self,
  filename 
)

Definition at line 829 of file lex.py.

00829 
00830     def validate_file(self,filename):
00831         import os.path
00832         base,ext = os.path.splitext(filename)
00833         if ext != '.py': return         # No idea what the file is. Return OK
00834 
00835         try:
00836             f = open(filename)
00837             lines = f.readlines()
00838             f.close()
00839         except IOError:
00840             return                      # Couldn't find the file.  Don't worry about it
00841 
00842         fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(')
00843         sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=')
00844 
00845         counthash = { }
00846         linen = 1
00847         for l in lines:
00848             m = fre.match(l)
00849             if not m:
00850                 m = sre.match(l)
00851             if m:
00852                 name = m.group(1)
00853                 prev = counthash.get(name)
00854                 if not prev:
00855                     counthash[name] = linen
00856                 else:
00857                     self.log.error("%s:%d: Rule %s redefined. Previously defined on line %d",filename,linen,name,prev)
00858                     self.error = 1
00859             linen += 1
00860             
00861 # -----------------------------------------------------------------------------
00862 # lex(module)
00863 #
00864 # Build all of the regular expression rules from definitions in the supplied module
# -----------------------------------------------------------------------------

Here is the caller graph for this function:

Definition at line 609 of file lex.py.

00609 
00610     def validate_literals(self):
00611         try:
00612             for c in self.literals:
00613                 if not isinstance(c,StringTypes) or len(c) > 1:
00614                     self.log.error("Invalid literal %s. Must be a single character", repr(c))
00615                     self.error = 1
00616                     continue
00617 
00618         except TypeError:
00619             self.log.error("Invalid literals specification. literals must be a sequence of characters")
00620             self.error = 1

Here is the caller graph for this function:

Definition at line 723 of file lex.py.

00723 
00724     def validate_rules(self):
00725         for state in self.stateinfo:
00726             # Validate all rules defined by functions
00727 
00728             
00729 
00730             for fname, f in self.funcsym[state]:
00731                 line = func_code(f).co_firstlineno
00732                 file = func_code(f).co_filename
00733                 self.files[file] = 1
00734 
00735                 tokname = self.toknames[fname]
00736                 if isinstance(f, types.MethodType):
00737                     reqargs = 2
00738                 else:
00739                     reqargs = 1
00740                 nargs = func_code(f).co_argcount
00741                 if nargs > reqargs:
00742                     self.log.error("%s:%d: Rule '%s' has too many arguments",file,line,f.__name__)
00743                     self.error = 1
00744                     continue
00745 
00746                 if nargs < reqargs:
00747                     self.log.error("%s:%d: Rule '%s' requires an argument", file,line,f.__name__)
00748                     self.error = 1
00749                     continue
00750 
00751                 if not f.__doc__:
00752                     self.log.error("%s:%d: No regular expression defined for rule '%s'",file,line,f.__name__)
00753                     self.error = 1
00754                     continue
00755 
00756                 try:
00757                     c = re.compile("(?P<%s>%s)" % (fname,f.__doc__), re.VERBOSE | self.reflags)
00758                     if c.match(""):
00759                         self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file,line,f.__name__)
00760                         self.error = 1
00761                 except re.error:
00762                     _etype, e, _etrace = sys.exc_info()
00763                     self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file,line,f.__name__,e)
00764                     if '#' in f.__doc__:
00765                         self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'",file,line, f.__name__)
00766                     self.error = 1
00767 
00768             # Validate all rules defined by strings
00769             for name,r in self.strsym[state]:
00770                 tokname = self.toknames[name]
00771                 if tokname == 'error':
00772                     self.log.error("Rule '%s' must be defined as a function", name)
00773                     self.error = 1
00774                     continue
00775 
00776                 if not tokname in self.tokens and tokname.find("ignore_") < 0:
00777                     self.log.error("Rule '%s' defined for an unspecified token %s",name,tokname)
00778                     self.error = 1
00779                     continue
00780 
00781                 try:
00782                     c = re.compile("(?P<%s>%s)" % (name,r),re.VERBOSE | self.reflags)
00783                     if (c.match("")):
00784                          self.log.error("Regular expression for rule '%s' matches empty string",name)
00785                          self.error = 1
00786                 except re.error:
00787                     _etype, e, _etrace = sys.exc_info()
00788                     self.log.error("Invalid regular expression for rule '%s'. %s",name,e)
00789                     if '#' in r:
00790                          self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'",name)
00791                     self.error = 1
00792 
00793             if not self.funcsym[state] and not self.strsym[state]:
00794                 self.log.error("No rules defined for state '%s'",state)
00795                 self.error = 1
00796 
00797             # Validate the error function
00798             efunc = self.errorf.get(state,None)
00799             if efunc:
00800                 f = efunc
00801                 line = func_code(f).co_firstlineno
00802                 file = func_code(f).co_filename
00803                 self.files[file] = 1
00804 
00805                 if isinstance(f, types.MethodType):
00806                     reqargs = 2
00807                 else:
00808                     reqargs = 1
00809                 nargs = func_code(f).co_argcount
00810                 if nargs > reqargs:
00811                     self.log.error("%s:%d: Rule '%s' has too many arguments",file,line,f.__name__)
00812                     self.error = 1
00813 
00814                 if nargs < reqargs:
00815                     self.log.error("%s:%d: Rule '%s' requires an argument", file,line,f.__name__)
00816                     self.error = 1
00817 
00818         for f in self.files:
00819             self.validate_file(f)
00820 

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 594 of file lex.py.

00594 
00595     def validate_tokens(self):
00596         terminals = {}
00597         for n in self.tokens:
00598             if not _is_identifier.match(n):
00599                 self.log.error("Bad token name '%s'",n)
00600                 self.error = 1
00601             if n in terminals:
00602                 self.log.warning("Token '%s' multiply defined", n)
00603             terminals[n] = 1

Here is the caller graph for this function:


Member Data Documentation

Definition at line 552 of file lex.py.

Definition at line 547 of file lex.py.

Definition at line 661 of file lex.py.

Definition at line 551 of file lex.py.

Definition at line 658 of file lex.py.

Definition at line 660 of file lex.py.

Definition at line 546 of file lex.py.

Definition at line 606 of file lex.py.

Definition at line 555 of file lex.py.

Definition at line 549 of file lex.py.

Definition at line 550 of file lex.py.

Definition at line 622 of file lex.py.

Definition at line 659 of file lex.py.

Definition at line 548 of file lex.py.

Definition at line 657 of file lex.py.


The documentation for this class was generated from the following file: