Back to index

moin  1.9.0~rc2
token.py
Go to the documentation of this file.
00001 # -*- coding: utf-8 -*-
00002 """
00003     pygments.token
00004     ~~~~~~~~~~~~~~
00005 
00006     Basic token types and the standard tokens.
00007 
00008     :copyright: Copyright 2006-2009 by the Pygments team, see AUTHORS.
00009     :license: BSD, see LICENSE for details.
00010 """
00011 try:
00012     set
00013 except NameError:
00014     from sets import Set as set
00015 
00016 
00017 class _TokenType(tuple):
00018     parent = None
00019 
00020     def split(self):
00021         buf = []
00022         node = self
00023         while node is not None:
00024             buf.append(node)
00025             node = node.parent
00026         buf.reverse()
00027         return buf
00028 
00029     def __init__(self, *args):
00030         # no need to call super.__init__
00031         self.subtypes = set()
00032 
00033     def __contains__(self, val):
00034         return self is val or (
00035             type(val) is self.__class__ and
00036             val[:len(self)] == self
00037         )
00038 
00039     def __getattr__(self, val):
00040         if not val or not val[0].isupper():
00041             return tuple.__getattribute__(self, val)
00042         new = _TokenType(self + (val,))
00043         setattr(self, val, new)
00044         self.subtypes.add(new)
00045         new.parent = self
00046         return new
00047 
00048     def __hash__(self):
00049         return hash(tuple(self))
00050 
00051     def __repr__(self):
00052         return 'Token' + (self and '.' or '') + '.'.join(self)
00053 
00054 
00055 Token       = _TokenType()
00056 
00057 # Special token types
00058 Text        = Token.Text
00059 Whitespace  = Text.Whitespace
00060 Error       = Token.Error
00061 # Text that doesn't belong to this lexer (e.g. HTML in PHP)
00062 Other       = Token.Other
00063 
00064 # Common token types for source code
00065 Keyword     = Token.Keyword
00066 Name        = Token.Name
00067 Literal     = Token.Literal
00068 String      = Literal.String
00069 Number      = Literal.Number
00070 Punctuation = Token.Punctuation
00071 Operator    = Token.Operator
00072 Comment     = Token.Comment
00073 
00074 # Generic types for non-source code
00075 Generic     = Token.Generic
00076 
00077 # String and some others are not direct childs of Token.
00078 # alias them:
00079 Token.Token = Token
00080 Token.String = String
00081 Token.Number = Number
00082 
00083 
00084 def is_token_subtype(ttype, other):
00085     """
00086     Return True if ``ttype`` is a subtype of ``other``.
00087 
00088     exists for backwards compatibility. use ``ttype in other`` now.
00089     """
00090     return ttype in other
00091 
00092 
00093 def string_to_tokentype(s):
00094     """
00095     Convert a string into a token type::
00096 
00097         >>> string_to_token('String.Double')
00098         Token.Literal.String.Double
00099         >>> string_to_token('Token.Literal.Number')
00100         Token.Literal.Number
00101         >>> string_to_token('')
00102         Token
00103 
00104     Tokens that are already tokens are returned unchanged:
00105 
00106         >>> string_to_token(String)
00107         Token.Literal.String
00108     """
00109     if isinstance(s, _TokenType):
00110         return s
00111     if not s:
00112         return Token
00113     node = Token
00114     for item in s.split('.'):
00115         node = getattr(node, item)
00116     return node
00117 
00118 
00119 # Map standard token types to short names, used in CSS class naming.
00120 # If you add a new item, please be sure to run this file to perform
00121 # a consistency check for duplicate values.
00122 STANDARD_TYPES = {
00123     Token:                         '',
00124 
00125     Text:                          '',
00126     Whitespace:                    'w',
00127     Error:                         'err',
00128     Other:                         'x',
00129 
00130     Keyword:                       'k',
00131     Keyword.Constant:              'kc',
00132     Keyword.Declaration:           'kd',
00133     Keyword.Namespace:             'kn',
00134     Keyword.Pseudo:                'kp',
00135     Keyword.Reserved:              'kr',
00136     Keyword.Type:                  'kt',
00137 
00138     Name:                          'n',
00139     Name.Attribute:                'na',
00140     Name.Builtin:                  'nb',
00141     Name.Builtin.Pseudo:           'bp',
00142     Name.Class:                    'nc',
00143     Name.Constant:                 'no',
00144     Name.Decorator:                'nd',
00145     Name.Entity:                   'ni',
00146     Name.Exception:                'ne',
00147     Name.Function:                 'nf',
00148     Name.Property:                 'py',
00149     Name.Label:                    'nl',
00150     Name.Namespace:                'nn',
00151     Name.Other:                    'nx',
00152     Name.Tag:                      'nt',
00153     Name.Variable:                 'nv',
00154     Name.Variable.Class:           'vc',
00155     Name.Variable.Global:          'vg',
00156     Name.Variable.Instance:        'vi',
00157 
00158     Literal:                       'l',
00159     Literal.Date:                  'ld',
00160 
00161     String:                        's',
00162     String.Backtick:               'sb',
00163     String.Char:                   'sc',
00164     String.Doc:                    'sd',
00165     String.Double:                 's2',
00166     String.Escape:                 'se',
00167     String.Heredoc:                'sh',
00168     String.Interpol:               'si',
00169     String.Other:                  'sx',
00170     String.Regex:                  'sr',
00171     String.Single:                 's1',
00172     String.Symbol:                 'ss',
00173 
00174     Number:                        'm',
00175     Number.Float:                  'mf',
00176     Number.Hex:                    'mh',
00177     Number.Integer:                'mi',
00178     Number.Integer.Long:           'il',
00179     Number.Oct:                    'mo',
00180 
00181     Operator:                      'o',
00182     Operator.Word:                 'ow',
00183 
00184     Punctuation:                   'p',
00185 
00186     Comment:                       'c',
00187     Comment.Multiline:             'cm',
00188     Comment.Preproc:               'cp',
00189     Comment.Single:                'c1',
00190     Comment.Special:               'cs',
00191 
00192     Generic:                       'g',
00193     Generic.Deleted:               'gd',
00194     Generic.Emph:                  'ge',
00195     Generic.Error:                 'gr',
00196     Generic.Heading:               'gh',
00197     Generic.Inserted:              'gi',
00198     Generic.Output:                'go',
00199     Generic.Prompt:                'gp',
00200     Generic.Strong:                'gs',
00201     Generic.Subheading:            'gu',
00202     Generic.Traceback:             'gt',
00203 }