cpp.tokenize Namespace Reference

Classes
class	Token

Functions
def	_GetChar (source, start, i)
def	_GetString (source, start, i)
def	GetTokens (source)
def	main (argv)

Variables
string	__author__ = 'nnorwitz@google.com (Neal Norwitz)'
string	_letters = 'abcdefghijklmnopqrstuvwxyz'
	_STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))
string	CONSTANT = 'CONSTANT'
	HEX_DIGITS = set('0123456789abcdefABCDEF')
	INT_OR_FLOAT_DIGITS = set('01234567890eE-+')
string	NAME = 'NAME'
string	PREPROCESSOR = 'PREPROCESSOR'
string	SYNTAX = 'SYNTAX'
string	UNKNOWN = 'UNKNOWN'
	VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$')
	WHENCE_QUEUE
	WHENCE_STREAM

Function Documentation

def cpp.tokenize._GetChar (   source,   start,   i  )

private

Definition at line 105 of file tokenize.py.

def cpp.tokenize._GetString (   source,   start,   i  )

private

Definition at line 89 of file tokenize.py.

def cpp.tokenize.GetTokens

(

source

)

Returns a sequence of Tokens.

Args:
  source: string of C++ source code.

Yields:
  Token that represents the next token in the source.

Definition at line 119 of file tokenize.py.

def cpp.tokenize.main

(

argv

)

Driver mostly for testing purposes.

Definition at line 274 of file tokenize.py.

Variable Documentation

string cpp.tokenize.__author__ = 'nnorwitz@google.com (Neal Norwitz)'

private

Definition at line 20 of file tokenize.py.

string cpp.tokenize._letters = 'abcdefghijklmnopqrstuvwxyz'

private

Definition at line 42 of file tokenize.py.

cpp.tokenize._STR_PREFIXES = set(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))

private

Definition at line 49 of file tokenize.py.

string cpp.tokenize.CONSTANT = 'CONSTANT'

Definition at line 55 of file tokenize.py.

cpp.tokenize.HEX_DIGITS = set('0123456789abcdefABCDEF')

Definition at line 44 of file tokenize.py.

cpp.tokenize.INT_OR_FLOAT_DIGITS = set('01234567890eE-+')

Definition at line 45 of file tokenize.py.

string cpp.tokenize.NAME = 'NAME'

Definition at line 56 of file tokenize.py.

string cpp.tokenize.PREPROCESSOR = 'PREPROCESSOR'

Definition at line 57 of file tokenize.py.

string cpp.tokenize.SYNTAX = 'SYNTAX'

Definition at line 54 of file tokenize.py.

string cpp.tokenize.UNKNOWN = 'UNKNOWN'

Definition at line 53 of file tokenize.py.

cpp.tokenize.VALID_IDENTIFIER_CHARS = set(_letters + _letters.upper() + '_0123456789$')

Definition at line 43 of file tokenize.py.

cpp.tokenize.WHENCE_QUEUE

Definition at line 61 of file tokenize.py.

cpp.tokenize.WHENCE_STREAM

Definition at line 61 of file tokenize.py.