google.protobuf.text_format.Tokenizer Class Reference

Inheritance diagram for google.protobuf.text_format.Tokenizer:

Public Member Functions
def	__init__ (self, lines, skip_comments=True)
def	__init__ (self, lines, skip_comments=True)
def	AtEnd (self)
def	AtEnd (self)
def	Consume (self, token)
def	Consume (self, token)
def	ConsumeBool (self)
def	ConsumeBool (self)
def	ConsumeByteString (self)
def	ConsumeByteString (self)
def	ConsumeComment (self)
def	ConsumeComment (self)
def	ConsumeCommentOrTrailingComment (self)
def	ConsumeCommentOrTrailingComment (self)
def	ConsumeEnum (self, field)
def	ConsumeEnum (self, field)
def	ConsumeFloat (self)
def	ConsumeFloat (self)
def	ConsumeIdentifier (self)
def	ConsumeIdentifier (self)
def	ConsumeIdentifierOrNumber (self)
def	ConsumeIdentifierOrNumber (self)
def	ConsumeInteger (self)
def	ConsumeInteger (self, is_long=False)
def	ConsumeString (self)
def	ConsumeString (self)
def	LookingAt (self, token)
def	LookingAt (self, token)
def	NextToken (self)
def	NextToken (self)
def	ParseError (self, message)
def	ParseError (self, message)
def	ParseErrorPreviousToken (self, message)
def	ParseErrorPreviousToken (self, message)
def	TryConsume (self, token)
def	TryConsume (self, token)
def	TryConsumeByteString (self)
def	TryConsumeByteString (self)
def	TryConsumeFloat (self)
def	TryConsumeFloat (self)
def	TryConsumeIdentifier (self)
def	TryConsumeIdentifier (self)
def	TryConsumeIdentifierOrNumber (self)
def	TryConsumeIdentifierOrNumber (self)
def	TryConsumeInteger (self)
def	TryConsumeInteger (self)

Public Attributes
	token

Private Member Functions
def	_ConsumeSingleByteString (self)
def	_ConsumeSingleByteString (self)
def	_PopLine (self)
def	_PopLine (self)
def	_SkipWhitespace (self)
def	_SkipWhitespace (self)
def	_StringParseError (self, e)
def	_StringParseError (self, e)

Private Attributes
	_column
	_current_line
	_line
	_lines
	_more_lines
	_position
	_previous_column
	_previous_line
	_skip_comments
	_token_start
	_whitespace_pattern

Static Private Attributes
	_COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE)
	_IDENTIFIER = re.compile(r'[^\d\W]\w*')
	_IDENTIFIER_OR_NUMBER = re.compile(r'\w+')
	_TOKEN
	_WHITESPACE = re.compile(r'\s+')
	_WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE)

Detailed Description

Protocol buffer text representation tokenizer.

This class handles the lower level string parsing by splitting it into
meaningful tokens.

It was directly ported from the Java protocol buffer API.

Definition at line 1181 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

Constructor & Destructor Documentation

__init__() [1/2]

def google.protobuf.text_format.Tokenizer.__init__	(		self,
			lines,
			skip_comments = True
	)

Definition at line 1205 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

__init__() [2/2]

def google.protobuf.text_format.Tokenizer.__init__	(		self,
			lines,
			skip_comments = True
	)

Definition at line 1241 of file protobuf/python/google/protobuf/text_format.py.

Member Function Documentation

_ConsumeSingleByteString() [1/2]

def google.protobuf.text_format.Tokenizer._ConsumeSingleByteString (   self )

private

Consume one token of a string literal.

String literals (whether bytes or text) can come in multiple adjacent
tokens which are automatically concatenated, like in C or Python.  This
method only consumes one token.

Returns:
  The token parsed.
Raises:
  ParseError: When the wrong format data is found.

Definition at line 1448 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_ConsumeSingleByteString() [2/2]

def google.protobuf.text_format.Tokenizer._ConsumeSingleByteString (   self )

private

Consume one token of a string literal.

String literals (whether bytes or text) can come in multiple adjacent
tokens which are automatically concatenated, like in C or Python.  This
method only consumes one token.

Returns:
  The token parsed.
Raises:
  ParseError: When the wrong format data is found.

Definition at line 1481 of file protobuf/python/google/protobuf/text_format.py.

_PopLine() [1/2]

def google.protobuf.text_format.Tokenizer._PopLine (   self )

private

Definition at line 1233 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_PopLine() [2/2]

def google.protobuf.text_format.Tokenizer._PopLine (   self )

private

Definition at line 1269 of file protobuf/python/google/protobuf/text_format.py.

_SkipWhitespace() [1/2]

def google.protobuf.text_format.Tokenizer._SkipWhitespace (   self )

private

Definition at line 1245 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_SkipWhitespace() [2/2]

def google.protobuf.text_format.Tokenizer._SkipWhitespace (   self )

private

Definition at line 1281 of file protobuf/python/google/protobuf/text_format.py.

_StringParseError() [1/2]

def google.protobuf.text_format.Tokenizer._StringParseError (   self,   e  )

private

Definition at line 1499 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_StringParseError() [2/2]

def google.protobuf.text_format.Tokenizer._StringParseError (   self,   e  )

private

Definition at line 1532 of file protobuf/python/google/protobuf/text_format.py.

AtEnd() [1/2]

def google.protobuf.text_format.Tokenizer.AtEnd

(

self

)

Checks the end of the text was reached.

Returns:
  True iff the end was reached.

Definition at line 1225 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

AtEnd() [2/2]

def google.protobuf.text_format.Tokenizer.AtEnd

(

self

)

Checks the end of the text was reached.

Returns:
  True iff the end was reached.

Definition at line 1261 of file protobuf/python/google/protobuf/text_format.py.

Consume() [1/2]

def google.protobuf.text_format.Tokenizer.Consume	(		self,
			token
	)

Consumes a piece of text.

Args:
  token: Text to consume.

Raises:
  ParseError: If the text couldn't be consumed.

Definition at line 1268 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

Consume() [2/2]

def google.protobuf.text_format.Tokenizer.Consume	(		self,
			token
	)

Consumes a piece of text.

Args:
  token: Text to consume.

Raises:
  ParseError: If the text couldn't be consumed.

Definition at line 1304 of file protobuf/python/google/protobuf/text_format.py.

ConsumeBool() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeBool

(

self

)

Consumes a boolean value.

Returns:
  The bool parsed.

Raises:
  ParseError: If a boolean value couldn't be consumed.

Definition at line 1396 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeBool() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeBool

(

self

)

Consumes a boolean value.

Returns:
  The bool parsed.

Raises:
  ParseError: If a boolean value couldn't be consumed.

Definition at line 1429 of file protobuf/python/google/protobuf/text_format.py.

ConsumeByteString() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeByteString

(

self

)

Consumes a byte array value.

Returns:
  The array parsed (as a string).

Raises:
  ParseError: If a byte array value couldn't be consumed.

Definition at line 1434 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeByteString() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeByteString

(

self

)

Consumes a byte array value.

Returns:
  The array parsed (as a string).

Raises:
  ParseError: If a byte array value couldn't be consumed.

Definition at line 1467 of file protobuf/python/google/protobuf/text_format.py.

ConsumeComment() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeComment

(

self

)

Definition at line 1280 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeComment() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeComment

(

self

)

Definition at line 1316 of file protobuf/python/google/protobuf/text_format.py.

ConsumeCommentOrTrailingComment() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeCommentOrTrailingComment

(

self

)

Consumes a comment, returns a 2-tuple (trailing bool, comment str).

Definition at line 1287 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeCommentOrTrailingComment() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeCommentOrTrailingComment

(

self

)

Consumes a comment, returns a 2-tuple (trailing bool, comment str).

Definition at line 1323 of file protobuf/python/google/protobuf/text_format.py.

ConsumeEnum() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeEnum	(		self,
			field
	)

Definition at line 1474 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeEnum() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeEnum	(		self,
			field
	)

Definition at line 1507 of file protobuf/python/google/protobuf/text_format.py.

ConsumeFloat() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeFloat

(

self

)

Consumes an floating point number.

Returns:
  The number parsed.

Raises:
  ParseError: If a floating point number couldn't be consumed.

Definition at line 1380 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeFloat() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeFloat

(

self

)

Consumes an floating point number.

Returns:
  The number parsed.

Raises:
  ParseError: If a floating point number couldn't be consumed.

Definition at line 1413 of file protobuf/python/google/protobuf/text_format.py.

ConsumeIdentifier() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeIdentifier

(

self

)

Consumes protocol message field identifier.

Returns:
  Identifier string.

Raises:
  ParseError: If an identifier couldn't be consumed.

Definition at line 1310 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeIdentifier() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeIdentifier

(

self

)

Consumes protocol message field identifier.

Returns:
  Identifier string.

Raises:
  ParseError: If an identifier couldn't be consumed.

Definition at line 1346 of file protobuf/python/google/protobuf/text_format.py.

ConsumeIdentifierOrNumber() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeIdentifierOrNumber

(

self

)

Consumes protocol message field identifier.

Returns:
  Identifier string.

Raises:
  ParseError: If an identifier couldn't be consumed.

Definition at line 1332 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeIdentifierOrNumber() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeIdentifierOrNumber

(

self

)

Consumes protocol message field identifier.

Returns:
  Identifier string.

Raises:
  ParseError: If an identifier couldn't be consumed.

Definition at line 1368 of file protobuf/python/google/protobuf/text_format.py.

ConsumeInteger() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeInteger

(

self

)

Consumes an integer number.

Returns:
  The integer parsed.

Raises:
  ParseError: If an integer couldn't be consumed.

Definition at line 1390 of file protobuf/python/google/protobuf/text_format.py.

ConsumeInteger() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeInteger	(		self,
			is_long = False
	)

Consumes an integer number.

Args:
  is_long: True if the value should be returned as a long integer.
Returns:
  The integer parsed.

Raises:
  ParseError: If an integer couldn't be consumed.

Definition at line 1355 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeString() [1/2]

def google.protobuf.text_format.Tokenizer.ConsumeString

(

self

)

Consumes a string value.

Returns:
  The string parsed.

Raises:
  ParseError: If a string value couldn't be consumed.

Definition at line 1419 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ConsumeString() [2/2]

def google.protobuf.text_format.Tokenizer.ConsumeString

(

self

)

Consumes a string value.

Returns:
  The string parsed.

Raises:
  ParseError: If a string value couldn't be consumed.

Definition at line 1452 of file protobuf/python/google/protobuf/text_format.py.

LookingAt() [1/2]

def google.protobuf.text_format.Tokenizer.LookingAt	(		self,
			token
	)

Definition at line 1222 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

LookingAt() [2/2]

def google.protobuf.text_format.Tokenizer.LookingAt	(		self,
			token
	)

Definition at line 1258 of file protobuf/python/google/protobuf/text_format.py.

NextToken() [1/2]

def google.protobuf.text_format.Tokenizer.NextToken

(

self

)

Reads the next meaningful token.

Definition at line 1502 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

NextToken() [2/2]

def google.protobuf.text_format.Tokenizer.NextToken

(

self

)

Reads the next meaningful token.

Definition at line 1535 of file protobuf/python/google/protobuf/text_format.py.

ParseError() [1/2]

def google.protobuf.text_format.Tokenizer.ParseError	(		self,
			message
	)

Creates and *returns* a ParseError for the current token.

Definition at line 1494 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ParseError() [2/2]

def google.protobuf.text_format.Tokenizer.ParseError	(		self,
			message
	)

Creates and *returns* a ParseError for the current token.

Definition at line 1527 of file protobuf/python/google/protobuf/text_format.py.

ParseErrorPreviousToken() [1/2]

def google.protobuf.text_format.Tokenizer.ParseErrorPreviousToken	(		self,
			message
	)

Creates and *returns* a ParseError for the previously read token.

Args:
  message: A message to set for the exception.

Returns:
  A ParseError instance.

Definition at line 1482 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

ParseErrorPreviousToken() [2/2]

def google.protobuf.text_format.Tokenizer.ParseErrorPreviousToken	(		self,
			message
	)

Creates and *returns* a ParseError for the previously read token.

Args:
  message: A message to set for the exception.

Returns:
  A ParseError instance.

Definition at line 1515 of file protobuf/python/google/protobuf/text_format.py.

TryConsume() [1/2]

def google.protobuf.text_format.Tokenizer.TryConsume	(		self,
			token
	)

Tries to consume a given piece of text.

Args:
  token: Text to consume.

Returns:
  True iff the text was consumed.

Definition at line 1254 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

TryConsume() [2/2]

def google.protobuf.text_format.Tokenizer.TryConsume	(		self,
			token
	)

Tries to consume a given piece of text.

Args:
  token: Text to consume.

Returns:
  True iff the text was consumed.

Definition at line 1290 of file protobuf/python/google/protobuf/text_format.py.

TryConsumeByteString() [1/2]

def google.protobuf.text_format.Tokenizer.TryConsumeByteString

(

self

)

Definition at line 1412 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

TryConsumeByteString() [2/2]

def google.protobuf.text_format.Tokenizer.TryConsumeByteString

(

self

)

Definition at line 1445 of file protobuf/python/google/protobuf/text_format.py.

TryConsumeFloat() [1/2]

def google.protobuf.text_format.Tokenizer.TryConsumeFloat

(

self

)

Definition at line 1373 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

TryConsumeFloat() [2/2]

def google.protobuf.text_format.Tokenizer.TryConsumeFloat

(

self

)

Definition at line 1406 of file protobuf/python/google/protobuf/text_format.py.

TryConsumeIdentifier() [1/2]

def google.protobuf.text_format.Tokenizer.TryConsumeIdentifier

(

self

)

Definition at line 1303 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

TryConsumeIdentifier() [2/2]

def google.protobuf.text_format.Tokenizer.TryConsumeIdentifier

(

self

)

Definition at line 1339 of file protobuf/python/google/protobuf/text_format.py.

TryConsumeIdentifierOrNumber() [1/2]

def google.protobuf.text_format.Tokenizer.TryConsumeIdentifierOrNumber

(

self

)

Definition at line 1325 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

TryConsumeIdentifierOrNumber() [2/2]

def google.protobuf.text_format.Tokenizer.TryConsumeIdentifierOrNumber

(

self

)

Definition at line 1361 of file protobuf/python/google/protobuf/text_format.py.

TryConsumeInteger() [1/2]

def google.protobuf.text_format.Tokenizer.TryConsumeInteger

(

self

)

Definition at line 1347 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

TryConsumeInteger() [2/2]

def google.protobuf.text_format.Tokenizer.TryConsumeInteger

(

self

)

Definition at line 1383 of file protobuf/python/google/protobuf/text_format.py.

Member Data Documentation

_column

google.protobuf.text_format.Tokenizer._column

private

Definition at line 1208 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_COMMENT

google.protobuf.text_format.Tokenizer._COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE)

staticprivate

Definition at line 1191 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_current_line

google.protobuf.text_format.Tokenizer._current_line

private

Definition at line 1212 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_IDENTIFIER

google.protobuf.text_format.Tokenizer._IDENTIFIER = re.compile(r'[^\d\W]\w*')

staticprivate

Definition at line 1202 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_IDENTIFIER_OR_NUMBER

google.protobuf.text_format.Tokenizer._IDENTIFIER_OR_NUMBER = re.compile(r'\w+')

staticprivate

Definition at line 1203 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_line

google.protobuf.text_format.Tokenizer._line

private

Definition at line 1207 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_lines

google.protobuf.text_format.Tokenizer._lines

private

Definition at line 1211 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_more_lines

google.protobuf.text_format.Tokenizer._more_lines

private

Definition at line 1215 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_position

google.protobuf.text_format.Tokenizer._position

private

Definition at line 1206 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_previous_column

google.protobuf.text_format.Tokenizer._previous_column

private

Definition at line 1214 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_previous_line

google.protobuf.text_format.Tokenizer._previous_line

private

Definition at line 1213 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_skip_comments

google.protobuf.text_format.Tokenizer._skip_comments

private

Definition at line 1216 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_TOKEN

google.protobuf.text_format.Tokenizer._TOKEN

staticprivate

Initial value:

=  re.compile('|'.join([
      r'[a-zA-Z_][0-9a-zA-Z_+-]*',  # an identifier
      r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*',  # a number
  ] + [  # quoted str for each quote mark
      # Avoid backtracking! https://stackoverflow.com/a/844267
      r'{qt}[^{qt}\n\\]*((\\.)+[^{qt}\n\\]*)*({qt}|\\?$)'.format(qt=mark)
      for mark in _QUOTES
  ]))

Definition at line 1193 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_token_start

google.protobuf.text_format.Tokenizer._token_start

private

Definition at line 1209 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_WHITESPACE

google.protobuf.text_format.Tokenizer._WHITESPACE = re.compile(r'\s+')

staticprivate

Definition at line 1190 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_WHITESPACE_OR_COMMENT

google.protobuf.text_format.Tokenizer._WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE)

staticprivate

Definition at line 1192 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

_whitespace_pattern

google.protobuf.text_format.Tokenizer._whitespace_pattern

private

Definition at line 1217 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

token

google.protobuf.text_format.Tokenizer.token

Definition at line 1210 of file bloaty/third_party/protobuf/python/google/protobuf/text_format.py.

---

The documentation for this class was generated from the following file:

• bloaty/third_party/protobuf/python/google/protobuf/text_format.py