Classes | Public Types | Public Member Functions | Static Public Member Functions | Private Types | Private Member Functions | Private Attributes | Static Private Attributes | List of all members
google::protobuf::io::Tokenizer Class Reference

#include <tokenizer.h>

Classes

struct  Token
 

Public Types

enum  CommentStyle { CPP_COMMENT_STYLE, SH_COMMENT_STYLE }
 
enum  TokenType {
  TYPE_START, TYPE_END, TYPE_IDENTIFIER, TYPE_INTEGER,
  TYPE_FLOAT, TYPE_STRING, TYPE_SYMBOL
}
 

Public Member Functions

const Token & current ()
 
bool Next ()
 
bool NextWithComments (std::string *prev_trailing_comments, std::vector< std::string > *detached_comments, std::string *next_leading_comments)
 
const Token & previous ()
 
void set_allow_f_after_float (bool value)
 
void set_allow_multiline_strings (bool allow)
 
void set_comment_style (CommentStyle style)
 
void set_require_space_after_number (bool require)
 
 Tokenizer (ZeroCopyInputStream *input, ErrorCollector *error_collector)
 
 ~Tokenizer ()
 

Static Public Member Functions

static bool IsIdentifier (const std::string &text)
 
static double ParseFloat (const std::string &text)
 
static bool ParseInteger (const std::string &text, uint64 max_value, uint64 *output)
 
static void ParseString (const std::string &text, std::string *output)
 
static void ParseStringAppend (const std::string &text, std::string *output)
 

Private Types

enum  NextCommentStatus { LINE_COMMENT, BLOCK_COMMENT, SLASH_NOT_COMMENT, NO_COMMENT }
 

Private Member Functions

void AddError (const std::string &message)
 
void ConsumeBlockComment (std::string *content)
 
void ConsumeLineComment (std::string *content)
 
TokenType ConsumeNumber (bool started_with_zero, bool started_with_dot)
 
template<typename CharacterClass >
void ConsumeOneOrMore (const char *error)
 
void ConsumeString (char delimiter)
 
template<typename CharacterClass >
void ConsumeZeroOrMore ()
 
void EndToken ()
 
 GOOGLE_DISALLOW_EVIL_CONSTRUCTORS (Tokenizer)
 
template<typename CharacterClass >
bool LookingAt ()
 
void NextChar ()
 
void RecordTo (std::string *target)
 
void Refresh ()
 
void StartToken ()
 
void StopRecording ()
 
bool TryConsume (char c)
 
NextCommentStatus TryConsumeCommentStart ()
 
template<typename CharacterClass >
bool TryConsumeOne ()
 

Private Attributes

bool allow_f_after_float_
 
bool allow_multiline_strings_
 
const char * buffer_
 
int buffer_pos_
 
int buffer_size_
 
ColumnNumber column_
 
CommentStyle comment_style_
 
Token current_
 
char current_char_
 
ErrorCollector * error_collector_
 
ZeroCopyInputStream * input_
 
int line_
 
Token previous_
 
bool read_error_
 
int record_start_
 
std::string * record_target_
 
bool require_space_after_number_
 

Static Private Attributes

static const int kTabWidth = 8
 

Detailed Description

Definition at line 93 of file tokenizer.h.

Member Enumeration Documentation

◆ CommentStyle

enum google::protobuf::io::Tokenizer::CommentStyle
Enumerator
CPP_COMMENT_STYLE 
SH_COMMENT_STYLE 

Definition at line 231 of file tokenizer.h.

◆ NextCommentStatus

enum google::protobuf::io::Tokenizer::NextCommentStatus
private
Enumerator
LINE_COMMENT 
BLOCK_COMMENT 
SLASH_NOT_COMMENT 
NO_COMMENT 

Definition at line 343 of file tokenizer.h.

◆ TokenType

enum google::protobuf::io::Tokenizer::TokenType
Enumerator
TYPE_START 
TYPE_END 
TYPE_IDENTIFIER 
TYPE_INTEGER 
TYPE_FLOAT 
TYPE_STRING 
TYPE_SYMBOL 

Definition at line 101 of file tokenizer.h.

Constructor & Destructor Documentation

◆ Tokenizer()

google::protobuf::io::Tokenizer::Tokenizer ( ZeroCopyInputStream *  input,
ErrorCollector *  error_collector 
)

Definition at line 194 of file tokenizer.cc.

◆ ~Tokenizer()

google::protobuf::io::Tokenizer::~Tokenizer ( )

Definition at line 218 of file tokenizer.cc.

Member Function Documentation

◆ AddError()

void google::protobuf::io::Tokenizer::AddError ( const std::string &  message)
inlineprivate

Definition at line 316 of file tokenizer.h.

◆ ConsumeBlockComment()

void google::protobuf::io::Tokenizer::ConsumeBlockComment ( std::string *  content)
private

Definition at line 492 of file tokenizer.cc.

◆ ConsumeLineComment()

void google::protobuf::io::Tokenizer::ConsumeLineComment ( std::string *  content)
private

Definition at line 481 of file tokenizer.cc.

◆ ConsumeNumber()

Tokenizer::TokenType google::protobuf::io::Tokenizer::ConsumeNumber ( bool  started_with_zero,
bool  started_with_dot 
)
private

Definition at line 426 of file tokenizer.cc.

◆ ConsumeOneOrMore()

template<typename CharacterClass >
void google::protobuf::io::Tokenizer::ConsumeOneOrMore ( const char *  error)
inlineprivate

Definition at line 347 of file tokenizer.cc.

◆ ConsumeString()

void google::protobuf::io::Tokenizer::ConsumeString ( char  delimiter)
private

Definition at line 361 of file tokenizer.cc.

◆ ConsumeZeroOrMore()

template<typename CharacterClass >
void google::protobuf::io::Tokenizer::ConsumeZeroOrMore
inlineprivate

Definition at line 340 of file tokenizer.cc.

◆ current()

const Tokenizer::Token & google::protobuf::io::Tokenizer::current ( )
inline

Definition at line 396 of file tokenizer.h.

◆ EndToken()

void google::protobuf::io::Tokenizer::EndToken ( )
inlineprivate

Definition at line 307 of file tokenizer.cc.

◆ GOOGLE_DISALLOW_EVIL_CONSTRUCTORS()

google::protobuf::io::Tokenizer::GOOGLE_DISALLOW_EVIL_CONSTRUCTORS ( Tokenizer  )
private

◆ IsIdentifier()

bool google::protobuf::io::Tokenizer::IsIdentifier ( const std::string &  text)
static

Definition at line 1121 of file tokenizer.cc.

◆ LookingAt()

template<typename CharacterClass >
bool google::protobuf::io::Tokenizer::LookingAt
inlineprivate

Definition at line 316 of file tokenizer.cc.

◆ Next()

bool google::protobuf::io::Tokenizer::Next ( )

Definition at line 564 of file tokenizer.cc.

◆ NextChar()

void google::protobuf::io::Tokenizer::NextChar ( )
private

Definition at line 229 of file tokenizer.cc.

◆ NextWithComments()

bool google::protobuf::io::Tokenizer::NextWithComments ( std::string *  prev_trailing_comments,
std::vector< std::string > *  detached_comments,
std::string *  next_leading_comments 
)

Definition at line 761 of file tokenizer.cc.

◆ ParseFloat()

double google::protobuf::io::Tokenizer::ParseFloat ( const std::string &  text)
static

Definition at line 901 of file tokenizer.cc.

◆ ParseInteger()

bool google::protobuf::io::Tokenizer::ParseInteger ( const std::string &  text,
uint64  max_value,
uint64 *  output 
)
static

Definition at line 862 of file tokenizer.cc.

◆ ParseString()

void google::protobuf::io::Tokenizer::ParseString ( const std::string &  text,
std::string *  output 
)
inlinestatic

Definition at line 400 of file tokenizer.h.

◆ ParseStringAppend()

void google::protobuf::io::Tokenizer::ParseStringAppend ( const std::string &  text,
std::string *  output 
)
static

Definition at line 1034 of file tokenizer.cc.

◆ previous()

const Tokenizer::Token & google::protobuf::io::Tokenizer::previous ( )
inline

Definition at line 398 of file tokenizer.h.

◆ RecordTo()

void google::protobuf::io::Tokenizer::RecordTo ( std::string *  target)
inlineprivate

Definition at line 281 of file tokenizer.cc.

◆ Refresh()

void google::protobuf::io::Tokenizer::Refresh ( )
private

Definition at line 250 of file tokenizer.cc.

◆ set_allow_f_after_float()

void google::protobuf::io::Tokenizer::set_allow_f_after_float ( bool  value)
inline

Definition at line 228 of file tokenizer.h.

◆ set_allow_multiline_strings()

void google::protobuf::io::Tokenizer::set_allow_multiline_strings ( bool  allow)
inline

Definition at line 250 of file tokenizer.h.

◆ set_comment_style()

void google::protobuf::io::Tokenizer::set_comment_style ( CommentStyle  style)
inline

Definition at line 240 of file tokenizer.h.

◆ set_require_space_after_number()

void google::protobuf::io::Tokenizer::set_require_space_after_number ( bool  require)
inline

Definition at line 244 of file tokenizer.h.

◆ StartToken()

void google::protobuf::io::Tokenizer::StartToken ( )
inlineprivate

Definition at line 299 of file tokenizer.cc.

◆ StopRecording()

void google::protobuf::io::Tokenizer::StopRecording ( )
inlineprivate

Definition at line 286 of file tokenizer.cc.

◆ TryConsume()

bool google::protobuf::io::Tokenizer::TryConsume ( char  c)
inlineprivate

Definition at line 330 of file tokenizer.cc.

◆ TryConsumeCommentStart()

Tokenizer::NextCommentStatus google::protobuf::io::Tokenizer::TryConsumeCommentStart ( )
private

Definition at line 540 of file tokenizer.cc.

◆ TryConsumeOne()

template<typename CharacterClass >
bool google::protobuf::io::Tokenizer::TryConsumeOne
inlineprivate

Definition at line 321 of file tokenizer.cc.

Member Data Documentation

◆ allow_f_after_float_

bool google::protobuf::io::Tokenizer::allow_f_after_float_
private

Definition at line 285 of file tokenizer.h.

◆ allow_multiline_strings_

bool google::protobuf::io::Tokenizer::allow_multiline_strings_
private

Definition at line 288 of file tokenizer.h.

◆ buffer_

const char* google::protobuf::io::Tokenizer::buffer_
private

Definition at line 268 of file tokenizer.h.

◆ buffer_pos_

int google::protobuf::io::Tokenizer::buffer_pos_
private

Definition at line 270 of file tokenizer.h.

◆ buffer_size_

int google::protobuf::io::Tokenizer::buffer_size_
private

Definition at line 269 of file tokenizer.h.

◆ column_

ColumnNumber google::protobuf::io::Tokenizer::column_
private

Definition at line 275 of file tokenizer.h.

◆ comment_style_

CommentStyle google::protobuf::io::Tokenizer::comment_style_
private

Definition at line 286 of file tokenizer.h.

◆ current_

Token google::protobuf::io::Tokenizer::current_
private

Definition at line 261 of file tokenizer.h.

◆ current_char_

char google::protobuf::io::Tokenizer::current_char_
private

Definition at line 267 of file tokenizer.h.

◆ error_collector_

ErrorCollector* google::protobuf::io::Tokenizer::error_collector_
private

Definition at line 265 of file tokenizer.h.

◆ input_

ZeroCopyInputStream* google::protobuf::io::Tokenizer::input_
private

Definition at line 264 of file tokenizer.h.

◆ kTabWidth

const int google::protobuf::io::Tokenizer::kTabWidth = 8
staticprivate

Definition at line 293 of file tokenizer.h.

◆ line_

int google::protobuf::io::Tokenizer::line_
private

Definition at line 274 of file tokenizer.h.

◆ previous_

Token google::protobuf::io::Tokenizer::previous_
private

Definition at line 262 of file tokenizer.h.

◆ read_error_

bool google::protobuf::io::Tokenizer::read_error_
private

Definition at line 271 of file tokenizer.h.

◆ record_start_

int google::protobuf::io::Tokenizer::record_start_
private

Definition at line 282 of file tokenizer.h.

◆ record_target_

std::string* google::protobuf::io::Tokenizer::record_target_
private

Definition at line 281 of file tokenizer.h.

◆ require_space_after_number_

bool google::protobuf::io::Tokenizer::require_space_after_number_
private

Definition at line 287 of file tokenizer.h.

The documentation for this class was generated from the following files:

libaditof
Author(s):
autogenerated on Wed May 21 2025 02:07:10