tokenize.py File Reference

Go to the source code of this file.

Classes
class	cpp.tokenize.Token
Namespaces
namespace	cpp::tokenize
Functions
def	cpp::tokenize._GetChar
def	cpp::tokenize._GetString
def	cpp::tokenize.GetTokens
def	cpp::tokenize.main
Variables
string	cpp::tokenize::__author__ = 'nnorwitz@google.com (Neal Norwitz)'
string	cpp::tokenize._letters = 'abcdefghijklmnopqrstuvwxyz'
tuple	cpp::tokenize._STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))
string	cpp::tokenize.CONSTANT = 'CONSTANT'
tuple	cpp::tokenize.HEX_DIGITS = set('0123456789abcdefABCDEF')
tuple	cpp::tokenize.INT_OR_FLOAT_DIGITS = set('01234567890eE-+')
string	cpp::tokenize.NAME = 'NAME'
string	cpp::tokenize.PREPROCESSOR = 'PREPROCESSOR'
string	cpp::tokenize.SYNTAX = 'SYNTAX'
string	cpp::tokenize.UNKNOWN = 'UNKNOWN'
tuple	cpp::tokenize.VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$')