Inheritance diagram for google.protobuf.text_format.Tokenizer:

Public Member Functions
def	__init__ (self, lines, skip_comments=True)

def	AtEnd (self)

def	Consume (self, token)

def	ConsumeBool (self)

def	ConsumeByteString (self)

def	ConsumeComment (self)

def	ConsumeCommentOrTrailingComment (self)

def	ConsumeEnum (self, field)

def	ConsumeFloat (self)

def	ConsumeIdentifier (self)

def	ConsumeIdentifierOrNumber (self)

def	ConsumeInteger (self, is_long=False)

def	ConsumeString (self)

def	LookingAt (self, token)

def	NextToken (self)

def	ParseError (self, message)

def	ParseErrorPreviousToken (self, message)

def	TryConsume (self, token)

def	TryConsumeByteString (self)

def	TryConsumeFloat (self)

def	TryConsumeIdentifier (self)

def	TryConsumeIdentifierOrNumber (self)

def	TryConsumeInteger (self)

Public Attributes
	token

Private Member Functions
def	_ConsumeSingleByteString (self)

def	_PopLine (self)

def	_SkipWhitespace (self)

def	_StringParseError (self, e)

Private Attributes
	_column

	_current_line

	_line

	_lines

	_more_lines

	_position

	_previous_column

	_previous_line

	_skip_comments

	_token_start

	_whitespace_pattern

Static Private Attributes
	_COMMENT = re.compile(r'(\s#.$)', re.MULTILINE)

	_IDENTIFIER = re.compile(r'[^\d\W]\w*')

	_IDENTIFIER_OR_NUMBER = re.compile(r'\w+')

	_TOKEN

	_WHITESPACE = re.compile(r'\s+')

	_WHITESPACE_OR_COMMENT = re.compile(r'(\s\|(#.*$))+', re.MULTILINE)

Detailed Description

Protocol buffer text representation tokenizer.

This class handles the lower level string parsing by splitting it into
meaningful tokens.

It was directly ported from the Java protocol buffer API.

Definition at line 1178 of file text_format.py.

Constructor & Destructor Documentation

◆ init()

def google.protobuf.text_format.Tokenizer.__init__	(	self,
		lines,
		skip_comments = `True`
	)

Definition at line 1202 of file text_format.py.

Member Function Documentation

◆ _ConsumeSingleByteString()

def google.protobuf.text_format.Tokenizer._ConsumeSingleByteString ( self )

private

Consume one token of a string literal.

String literals (whether bytes or text) can come in multiple adjacent
tokens which are automatically concatenated, like in C or Python.  This
method only consumes one token.

Returns:
  The token parsed.
Raises:
  ParseError: When the wrong format data is found.

Definition at line 1445 of file text_format.py.

◆ _PopLine()

def google.protobuf.text_format.Tokenizer._PopLine ( self )

private

Definition at line 1230 of file text_format.py.

◆ _SkipWhitespace()

def google.protobuf.text_format.Tokenizer._SkipWhitespace ( self )

private

Definition at line 1242 of file text_format.py.

◆ _StringParseError()

def google.protobuf.text_format.Tokenizer._StringParseError	(	self,
		e
	)

private

Definition at line 1496 of file text_format.py.

◆ AtEnd()

def google.protobuf.text_format.Tokenizer.AtEnd ( self )

Checks the end of the text was reached.

Returns:
  True iff the end was reached.

Definition at line 1222 of file text_format.py.

◆ Consume()

def google.protobuf.text_format.Tokenizer.Consume	(	self,
		token
	)

Consumes a piece of text.

Args:
  token: Text to consume.

Raises:
  ParseError: If the text couldn't be consumed.

Definition at line 1265 of file text_format.py.

◆ ConsumeBool()

def google.protobuf.text_format.Tokenizer.ConsumeBool ( self )

Consumes a boolean value.

Returns:
  The bool parsed.

Raises:
  ParseError: If a boolean value couldn't be consumed.

Definition at line 1393 of file text_format.py.

◆ ConsumeByteString()

def google.protobuf.text_format.Tokenizer.ConsumeByteString ( self )

Consumes a byte array value.

Returns:
  The array parsed (as a string).

Raises:
  ParseError: If a byte array value couldn't be consumed.

Definition at line 1431 of file text_format.py.

◆ ConsumeComment()

def google.protobuf.text_format.Tokenizer.ConsumeComment ( self )

Definition at line 1277 of file text_format.py.

◆ ConsumeCommentOrTrailingComment()

def google.protobuf.text_format.Tokenizer.ConsumeCommentOrTrailingComment ( self )

Consumes a comment, returns a 2-tuple (trailing bool, comment str).

Definition at line 1284 of file text_format.py.

◆ ConsumeEnum()

def google.protobuf.text_format.Tokenizer.ConsumeEnum	(	self,
		field
	)

Definition at line 1471 of file text_format.py.

◆ ConsumeFloat()

def google.protobuf.text_format.Tokenizer.ConsumeFloat ( self )

Consumes an floating point number.

Returns:
  The number parsed.

Raises:
  ParseError: If a floating point number couldn't be consumed.

Definition at line 1377 of file text_format.py.

◆ ConsumeIdentifier()

def google.protobuf.text_format.Tokenizer.ConsumeIdentifier ( self )

Consumes protocol message field identifier.

Returns:
  Identifier string.

Raises:
  ParseError: If an identifier couldn't be consumed.

Definition at line 1307 of file text_format.py.

◆ ConsumeIdentifierOrNumber()

def google.protobuf.text_format.Tokenizer.ConsumeIdentifierOrNumber ( self )

Consumes protocol message field identifier.

Returns:
  Identifier string.

Raises:
  ParseError: If an identifier couldn't be consumed.

Definition at line 1329 of file text_format.py.

◆ ConsumeInteger()

def google.protobuf.text_format.Tokenizer.ConsumeInteger	(	self,
		is_long = `False`
	)

Consumes an integer number.

Args:
  is_long: True if the value should be returned as a long integer.
Returns:
  The integer parsed.

Raises:
  ParseError: If an integer couldn't be consumed.

Definition at line 1352 of file text_format.py.

◆ ConsumeString()

def google.protobuf.text_format.Tokenizer.ConsumeString ( self )

Consumes a string value.

Returns:
  The string parsed.

Raises:
  ParseError: If a string value couldn't be consumed.

Definition at line 1416 of file text_format.py.

◆ LookingAt()

def google.protobuf.text_format.Tokenizer.LookingAt	(	self,
		token
	)

Definition at line 1219 of file text_format.py.

◆ NextToken()

def google.protobuf.text_format.Tokenizer.NextToken ( self )

Reads the next meaningful token.

Definition at line 1499 of file text_format.py.

◆ ParseError()

def google.protobuf.text_format.Tokenizer.ParseError	(	self,
		message
	)

Creates and *returns* a ParseError for the current token.

Definition at line 1491 of file text_format.py.

◆ ParseErrorPreviousToken()

def google.protobuf.text_format.Tokenizer.ParseErrorPreviousToken	(	self,
		message
	)

Creates and *returns* a ParseError for the previously read token.

Args:
  message: A message to set for the exception.

Returns:
  A ParseError instance.

Definition at line 1479 of file text_format.py.

◆ TryConsume()

def google.protobuf.text_format.Tokenizer.TryConsume	(	self,
		token
	)

Tries to consume a given piece of text.

Args:
  token: Text to consume.

Returns:
  True iff the text was consumed.

Definition at line 1251 of file text_format.py.

◆ TryConsumeByteString()

def google.protobuf.text_format.Tokenizer.TryConsumeByteString ( self )

Definition at line 1409 of file text_format.py.

◆ TryConsumeFloat()

def google.protobuf.text_format.Tokenizer.TryConsumeFloat ( self )

Definition at line 1370 of file text_format.py.

◆ TryConsumeIdentifier()

def google.protobuf.text_format.Tokenizer.TryConsumeIdentifier ( self )

Definition at line 1300 of file text_format.py.

◆ TryConsumeIdentifierOrNumber()

def google.protobuf.text_format.Tokenizer.TryConsumeIdentifierOrNumber ( self )

Definition at line 1322 of file text_format.py.

◆ TryConsumeInteger()

def google.protobuf.text_format.Tokenizer.TryConsumeInteger ( self )

Definition at line 1344 of file text_format.py.

Member Data Documentation

◆ _column

google.protobuf.text_format.Tokenizer._column

private

Definition at line 1205 of file text_format.py.

◆ _COMMENT

google.protobuf.text_format.Tokenizer._COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE)

staticprivate

Definition at line 1188 of file text_format.py.

◆ _current_line

google.protobuf.text_format.Tokenizer._current_line

private

Definition at line 1209 of file text_format.py.

◆ _IDENTIFIER

google.protobuf.text_format.Tokenizer._IDENTIFIER = re.compile(r'[^\d\W]\w*')

staticprivate

Definition at line 1199 of file text_format.py.

◆ _IDENTIFIER_OR_NUMBER

google.protobuf.text_format.Tokenizer._IDENTIFIER_OR_NUMBER = re.compile(r'\w+')

staticprivate

Definition at line 1200 of file text_format.py.

◆ _line

google.protobuf.text_format.Tokenizer._line

private

Definition at line 1204 of file text_format.py.

◆ _lines

google.protobuf.text_format.Tokenizer._lines

private

Definition at line 1208 of file text_format.py.

◆ _more_lines

google.protobuf.text_format.Tokenizer._more_lines

private

Definition at line 1212 of file text_format.py.

◆ _position

google.protobuf.text_format.Tokenizer._position

private

Definition at line 1203 of file text_format.py.

◆ _previous_column

google.protobuf.text_format.Tokenizer._previous_column

private

Definition at line 1211 of file text_format.py.

◆ _previous_line

google.protobuf.text_format.Tokenizer._previous_line

private

Definition at line 1210 of file text_format.py.

◆ _skip_comments

google.protobuf.text_format.Tokenizer._skip_comments

private

Definition at line 1213 of file text_format.py.

◆ _TOKEN

google.protobuf.text_format.Tokenizer._TOKEN

staticprivate

Initial value:

=  re.compile('|'.join([
      r'[a-zA-Z_][0-9a-zA-Z_+-]*',  # an identifier
      r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*',  # a number
  ] + [  # quoted str for each quote mark
      # Avoid backtracking! https://stackoverflow.com/a/844267
      r'{qt}[^{qt}\n\\]*((\\.)+[^{qt}\n\\]*)*({qt}|\\?$)'.format(qt=mark)
      for mark in _QUOTES
  ]))

Definition at line 1190 of file text_format.py.

◆ _token_start

google.protobuf.text_format.Tokenizer._token_start

private

Definition at line 1206 of file text_format.py.

◆ _WHITESPACE

google.protobuf.text_format.Tokenizer._WHITESPACE = re.compile(r'\s+')

staticprivate

Definition at line 1187 of file text_format.py.

◆ _WHITESPACE_OR_COMMENT

google.protobuf.text_format.Tokenizer._WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE)

staticprivate

Definition at line 1189 of file text_format.py.

◆ _whitespace_pattern

google.protobuf.text_format.Tokenizer._whitespace_pattern

private

Definition at line 1214 of file text_format.py.

◆ token

google.protobuf.text_format.Tokenizer.token

Definition at line 1207 of file text_format.py.

The documentation for this class was generated from the following file:

text_format.py

Public Member Functions

Public Attributes

Private Member Functions

Private Attributes

Static Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ _ConsumeSingleByteString()

◆ _PopLine()

◆ _SkipWhitespace()

◆ _StringParseError()

◆ AtEnd()

◆ Consume()

◆ ConsumeBool()

◆ ConsumeByteString()

◆ ConsumeComment()

◆ ConsumeCommentOrTrailingComment()

◆ ConsumeEnum()

◆ ConsumeFloat()

◆ ConsumeIdentifier()

◆ ConsumeIdentifierOrNumber()

◆ ConsumeInteger()

◆ ConsumeString()

◆ LookingAt()

◆ NextToken()

◆ ParseError()

◆ ParseErrorPreviousToken()

◆ TryConsume()

◆ TryConsumeByteString()

◆ TryConsumeFloat()

◆ TryConsumeIdentifier()

◆ TryConsumeIdentifierOrNumber()

◆ TryConsumeInteger()

Member Data Documentation

◆ _column

◆ _COMMENT

◆ _current_line

◆ _IDENTIFIER

◆ _IDENTIFIER_OR_NUMBER

◆ _line

◆ _lines

◆ _more_lines

◆ _position

◆ _previous_column

◆ _previous_line

◆ _skip_comments

◆ _TOKEN

◆ _token_start

◆ _WHITESPACE

◆ _WHITESPACE_OR_COMMENT

◆ _whitespace_pattern

◆ token

◆ init()