tokenizer.cc File Reference

#include <google/protobuf/io/tokenizer.h>
#include <google/protobuf/stubs/common.h>
#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/stringprintf.h>
#include <google/protobuf/io/strtod.h>
#include <google/protobuf/io/zero_copy_stream.h>
#include <google/protobuf/stubs/strutil.h>
#include <google/protobuf/stubs/stl_util.h>

Include dependency graph for tokenizer.cc:

Go to the source code of this file.

Namespaces
	google
	google::protobuf
	google::protobuf::io

Macros
#define	CHARACTER_CLASS(NAME, EXPRESSION)

Functions
template<typename CharacterClass >
static bool	google::protobuf::io::AllInClass (const std::string &s)
static void	google::protobuf::io::AppendUTF8 (uint32 code_point, std::string *output)
static uint32	google::protobuf::io::AssembleUTF16 (uint32 head_surrogate, uint32 trail_surrogate)
static const char *	google::protobuf::io::FetchUnicodePoint (const char *ptr, uint32 *code_point)
static bool	google::protobuf::io::IsHeadSurrogate (uint32 code_point)
static bool	google::protobuf::io::IsTrailSurrogate (uint32 code_point)
static bool	google::protobuf::io::ReadHexDigits (const char *ptr, int len, uint32 *result)
static int	google::protobuf::io::UnicodeLength (char key)

Variables
static const uint32	google::protobuf::io::kMaxHeadSurrogate = 0xdc00
static const uint32	google::protobuf::io::kMaxTrailSurrogate = 0xe000
static const uint32	google::protobuf::io::kMinHeadSurrogate = 0xd800
static const uint32	google::protobuf::io::kMinTrailSurrogate = 0xdc00

Macro Definition Documentation

CHARACTER_CLASS

#define CHARACTER_CLASS	(		NAME,
			EXPRESSION
	)

Value:

  class NAME {                                                \
   public:                                                    \
    static inline bool InClass(char c) { return EXPRESSION; } \
  }

Definition at line 116 of file tokenizer.cc.

Variable Documentation

can_attach_to_prev_

bool can_attach_to_prev_

private

Definition at line 756 of file tokenizer.cc.

comment_buffer_

std::string comment_buffer_

private

Definition at line 745 of file tokenizer.cc.

detached_comments_

std::vector<std::string>* detached_comments_

private

Definition at line 742 of file tokenizer.cc.

has_comment_

bool has_comment_

private

Definition at line 749 of file tokenizer.cc.

is_line_comment_

bool is_line_comment_

private

Definition at line 752 of file tokenizer.cc.

next_leading_comments_

std::string* next_leading_comments_

private

Definition at line 743 of file tokenizer.cc.

prev_trailing_comments_

std::string* prev_trailing_comments_

private

Definition at line 741 of file tokenizer.cc.