encodedstreamtest.cpp
Go to the documentation of this file.
1 // Tencent is pleased to support the open source community by making RapidJSON available.
2 //
3 // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
4 //
5 // Licensed under the MIT License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
7 //
8 // http://opensource.org/licenses/MIT
9 //
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
14 
15 #include "unittest.h"
16 #include "rapidjson/filereadstream.h"
17 #include "rapidjson/filewritestream.h"
18 #include "rapidjson/encodedstream.h"
19 #include "rapidjson/stringbuffer.h"
20 #include "rapidjson/memorystream.h"
21 #include "rapidjson/memorybuffer.h"
22 
23 using namespace rapidjson;
24 
25 class EncodedStreamTest : public ::testing::Test {
26 public:
27  EncodedStreamTest() : json_(), length_() {}
28  virtual ~EncodedStreamTest();
29 
30  virtual void SetUp() {
31  json_ = ReadFile("utf8.json", true, &length_);
32  }
33 
34  virtual void TearDown() {
35  free(json_);
36  json_ = 0;
37  }
38 
39 private:
40  EncodedStreamTest(const EncodedStreamTest&);
41  EncodedStreamTest& operator=(const EncodedStreamTest&);
42 
43 protected:
44  static FILE* Open(const char* filename) {
45  const char *paths[] = {
46  "encodings",
47  "bin/encodings",
48  "../bin/encodings",
49  "../../bin/encodings",
50  "../../../bin/encodings"
51  };
52  char buffer[1024];
53  for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
54  sprintf(buffer, "%s/%s", paths[i], filename);
55  FILE *fp = fopen(buffer, "rb");
56  if (fp)
57  return fp;
58  }
59  return 0;
60  }
61 
62  static char *ReadFile(const char* filename, bool appendPath, size_t* outLength) {
63  FILE *fp = appendPath ? Open(filename) : fopen(filename, "rb");
64 
65  if (!fp) {
66  *outLength = 0;
67  return 0;
68  }
69 
70  fseek(fp, 0, SEEK_END);
71  *outLength = static_cast<size_t>(ftell(fp));
72  fseek(fp, 0, SEEK_SET);
73  char* buffer = static_cast<char*>(malloc(*outLength + 1));
74  size_t readLength = fread(buffer, 1, *outLength, fp);
75  buffer[readLength] = '\0';
76  fclose(fp);
77  return buffer;
78  }
79 
80  template <typename FileEncoding, typename MemoryEncoding>
81  void TestEncodedInputStream(const char* filename) {
82  // Test FileReadStream
83  {
84  char buffer[16];
85  FILE *fp = Open(filename);
86  ASSERT_TRUE(fp != 0);
87  FileReadStream fs(fp, buffer, sizeof(buffer));
88  EncodedInputStream<FileEncoding, FileReadStream> eis(fs);
89  StringStream s(json_);
90 
91  while (eis.Peek() != '\0') {
92  unsigned expected, actual;
93  EXPECT_TRUE(UTF8<>::Decode(s, &expected));
94  EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
95  EXPECT_EQ(expected, actual);
96  }
97  EXPECT_EQ('\0', s.Peek());
98  fclose(fp);
99  }
100 
101  // Test MemoryStream
102  {
103  size_t size;
104  char* data = ReadFile(filename, true, &size);
105  MemoryStream ms(data, size);
106  EncodedInputStream<FileEncoding, MemoryStream> eis(ms);
107  StringStream s(json_);
108 
109  while (eis.Peek() != '\0') {
110  unsigned expected, actual;
111  EXPECT_TRUE(UTF8<>::Decode(s, &expected));
112  EXPECT_TRUE(MemoryEncoding::Decode(eis, &actual));
113  EXPECT_EQ(expected, actual);
114  }
115  EXPECT_EQ('\0', s.Peek());
116  free(data);
117  EXPECT_EQ(size, eis.Tell());
118  }
119  }
120 
121  void TestAutoUTFInputStream(const char *filename, bool expectHasBOM) {
122  // Test FileReadStream
123  {
124  char buffer[16];
125  FILE *fp = Open(filename);
126  ASSERT_TRUE(fp != 0);
127  FileReadStream fs(fp, buffer, sizeof(buffer));
128  AutoUTFInputStream<unsigned, FileReadStream> eis(fs);
129  EXPECT_EQ(expectHasBOM, eis.HasBOM());
130  StringStream s(json_);
131  while (eis.Peek() != '\0') {
132  unsigned expected, actual;
133  EXPECT_TRUE(UTF8<>::Decode(s, &expected));
134  EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
135  EXPECT_EQ(expected, actual);
136  }
137  EXPECT_EQ('\0', s.Peek());
138  fclose(fp);
139  }
140 
141  // Test MemoryStream
142  {
143  size_t size;
144  char* data = ReadFile(filename, true, &size);
145  MemoryStream ms(data, size);
146  AutoUTFInputStream<unsigned, MemoryStream> eis(ms);
147  EXPECT_EQ(expectHasBOM, eis.HasBOM());
148  StringStream s(json_);
149 
150  while (eis.Peek() != '\0') {
151  unsigned expected, actual;
152  EXPECT_TRUE(UTF8<>::Decode(s, &expected));
153  EXPECT_TRUE(AutoUTF<unsigned>::Decode(eis, &actual));
154  EXPECT_EQ(expected, actual);
155  }
156  EXPECT_EQ('\0', s.Peek());
157  free(data);
158  EXPECT_EQ(size, eis.Tell());
159  }
160  }
161 
162  template <typename FileEncoding, typename MemoryEncoding>
163  void TestEncodedOutputStream(const char* expectedFilename, bool putBOM) {
164  // Test FileWriteStream
165  {
166  char filename[L_tmpnam];
167  FILE* fp = TempFile(filename);
168  char buffer[16];
169  FileWriteStream os(fp, buffer, sizeof(buffer));
170  EncodedOutputStream<FileEncoding, FileWriteStream> eos(os, putBOM);
171  StringStream s(json_);
172  while (s.Peek() != '\0') {
173  bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
174  EXPECT_TRUE(success);
175  }
176  eos.Flush();
177  fclose(fp);
178  EXPECT_TRUE(CompareFile(filename, expectedFilename));
179  remove(filename);
180  }
181 
182  // Test MemoryBuffer
183  {
184  MemoryBuffer mb;
185  EncodedOutputStream<FileEncoding, MemoryBuffer> eos(mb, putBOM);
186  StringStream s(json_);
187  while (s.Peek() != '\0') {
188  bool success = Transcoder<UTF8<>, MemoryEncoding>::Transcode(s, eos);
189  EXPECT_TRUE(success);
190  }
191  eos.Flush();
192  EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
193  }
194  }
195 
196  void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename) {
197  // Test FileWriteStream
198  {
199  char filename[L_tmpnam];
200  FILE* fp = TempFile(filename);
201 
202  char buffer[16];
203  FileWriteStream os(fp, buffer, sizeof(buffer));
204  AutoUTFOutputStream<unsigned, FileWriteStream> eos(os, type, putBOM);
205  StringStream s(json_);
206  while (s.Peek() != '\0') {
207  bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
208  EXPECT_TRUE(success);
209  }
210  eos.Flush();
211  fclose(fp);
212  EXPECT_TRUE(CompareFile(filename, expectedFilename));
213  remove(filename);
214  }
215 
216  // Test MemoryBuffer
217  {
218  MemoryBuffer mb;
219  AutoUTFOutputStream<unsigned, MemoryBuffer> eos(mb, type, putBOM);
220  StringStream s(json_);
221  while (s.Peek() != '\0') {
222  bool success = Transcoder<UTF8<>, AutoUTF<unsigned> >::Transcode(s, eos);
223  EXPECT_TRUE(success);
224  }
225  eos.Flush();
226  EXPECT_TRUE(CompareBufferFile(mb.GetBuffer(), mb.GetSize(), expectedFilename));
227  }
228  }
229 
230  bool CompareFile(const char* filename, const char* expectedFilename) {
231  size_t actualLength, expectedLength;
232  char* actualBuffer = ReadFile(filename, false, &actualLength);
233  char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
234  bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
235  free(actualBuffer);
236  free(expectedBuffer);
237  return ret;
238  }
239 
240  bool CompareBufferFile(const char* actualBuffer, size_t actualLength, const char* expectedFilename) {
241  size_t expectedLength;
242  char* expectedBuffer = ReadFile(expectedFilename, true, &expectedLength);
243  bool ret = (expectedLength == actualLength) && memcmp(expectedBuffer, actualBuffer, actualLength) == 0;
244  free(expectedBuffer);
245  return ret;
246  }
247 
248  char *json_;
249  size_t length_;
250 };
251 
252 EncodedStreamTest::~EncodedStreamTest() {}
253 
254 TEST_F(EncodedStreamTest, EncodedInputStream) {
255  TestEncodedInputStream<UTF8<>, UTF8<> >("utf8.json");
256  TestEncodedInputStream<UTF8<>, UTF8<> >("utf8bom.json");
257  TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16le.json");
258  TestEncodedInputStream<UTF16LE<>, UTF16<> >("utf16lebom.json");
259  TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16be.json");
260  TestEncodedInputStream<UTF16BE<>, UTF16<> >("utf16bebom.json");
261  TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32le.json");
262  TestEncodedInputStream<UTF32LE<>, UTF32<> >("utf32lebom.json");
263  TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32be.json");
264  TestEncodedInputStream<UTF32BE<>, UTF32<> >("utf32bebom.json");
265 }
266 
267 TEST_F(EncodedStreamTest, AutoUTFInputStream) {
268  TestAutoUTFInputStream("utf8.json", false);
269  TestAutoUTFInputStream("utf8bom.json", true);
270  TestAutoUTFInputStream("utf16le.json", false);
271  TestAutoUTFInputStream("utf16lebom.json",true);
272  TestAutoUTFInputStream("utf16be.json", false);
273  TestAutoUTFInputStream("utf16bebom.json",true);
274  TestAutoUTFInputStream("utf32le.json", false);
275  TestAutoUTFInputStream("utf32lebom.json",true);
276  TestAutoUTFInputStream("utf32be.json", false);
277  TestAutoUTFInputStream("utf32bebom.json", true);
278 
279  {
280  // Auto detection fail, use user defined UTF type
281  const char json[] = "{ }";
282  MemoryStream ms(json, sizeof(json));
283  AutoUTFInputStream<unsigned, MemoryStream> eis(ms, kUTF8);
284  EXPECT_FALSE(eis.HasBOM());
285  EXPECT_EQ(kUTF8, eis.GetType());
286  }
287 }
288 
289 TEST_F(EncodedStreamTest, EncodedOutputStream) {
290  TestEncodedOutputStream<UTF8<>, UTF8<> >("utf8.json", false);
291  TestEncodedOutputStream<UTF8<>, UTF8<> >("utf8bom.json", true);
292  TestEncodedOutputStream<UTF16LE<>, UTF16<> >("utf16le.json", false);
293  TestEncodedOutputStream<UTF16LE<>, UTF16<> >("utf16lebom.json",true);
294  TestEncodedOutputStream<UTF16BE<>, UTF16<> >("utf16be.json", false);
295  TestEncodedOutputStream<UTF16BE<>, UTF16<> >("utf16bebom.json",true);
296  TestEncodedOutputStream<UTF32LE<>, UTF32<> >("utf32le.json", false);
297  TestEncodedOutputStream<UTF32LE<>, UTF32<> >("utf32lebom.json",true);
298  TestEncodedOutputStream<UTF32BE<>, UTF32<> >("utf32be.json", false);
299  TestEncodedOutputStream<UTF32BE<>, UTF32<> >("utf32bebom.json",true);
300 }
301 
302 TEST_F(EncodedStreamTest, AutoUTFOutputStream) {
303  TestAutoUTFOutputStream(kUTF8, false, "utf8.json");
304  TestAutoUTFOutputStream(kUTF8, true, "utf8bom.json");
305  TestAutoUTFOutputStream(kUTF16LE, false, "utf16le.json");
306  TestAutoUTFOutputStream(kUTF16LE, true, "utf16lebom.json");
307  TestAutoUTFOutputStream(kUTF16BE, false, "utf16be.json");
308  TestAutoUTFOutputStream(kUTF16BE, true, "utf16bebom.json");
309  TestAutoUTFOutputStream(kUTF32LE, false, "utf32le.json");
310  TestAutoUTFOutputStream(kUTF32LE, true, "utf32lebom.json");
311  TestAutoUTFOutputStream(kUTF32BE, false, "utf32be.json");
312  TestAutoUTFOutputStream(kUTF32BE, true, "utf32bebom.json");
313 }
Transcoder
Encoding conversion.
Definition: encodings.h:658
UTFType
UTFType
Runtime-specified UTF encoding type of a stream.
Definition: encodings.h:603
AutoUTFInputStream::HasBOM
bool HasBOM() const
Definition: encodedstream.h:154
MemoryStream
Represents an in-memory input byte stream.
Definition: memorystream.h:40
GenericStringStream
Read-only string stream.
Definition: fwd.h:47
kUTF32BE
UTF-32 big endian.
Definition: encodings.h:608
GenericMemoryBuffer
Represents an in-memory output byte stream.
Definition: fwd.h:74
kUTF16LE
UTF-16 little endian.
Definition: encodings.h:605
s
XmlRpcServer s
EncodedOutputStream
Output byte stream wrapper with statically bound encoding.
Definition: encodedstream.h:100
kUTF8
UTF-8.
Definition: encodings.h:604
UTF16
UTF-16 encoding.
Definition: encodings.h:269
TempFile
FILE * TempFile(char *filename)
Definition: unittest.h:80
FileWriteStream
Wrapper of C file stream for input using fread().
Definition: filewritestream.h:32
EncodedStreamTest::TestEncodedInputStream
void TestEncodedInputStream(const char *filename)
Definition: encodedstreamtest.cpp:81
EncodedStreamTest::TestAutoUTFInputStream
void TestAutoUTFInputStream(const char *filename, bool expectHasBOM)
Definition: encodedstreamtest.cpp:121
kUTF16BE
UTF-16 big endian.
Definition: encodings.h:606
GenericMemoryBuffer::GetSize
size_t GetSize() const
Definition: memorybuffer.h:54
GenericStringStream::Peek
Ch Peek() const
Definition: stream.h:115
AutoUTFOutputStream
Output stream wrapper with dynamically bound encoding and automatic encoding detection.
Definition: encodedstream.h:233
UTF8
UTF-8 encoding.
Definition: encodings.h:96
TEST_F
TEST_F(EncodedStreamTest, EncodedInputStream)
Definition: encodedstreamtest.cpp:254
AutoUTF
Dynamically select encoding according to stream&#39;s runtime-specified UTF encoding type.
Definition: encodings.h:615
EncodedInputStream::Peek
Ch Peek() const
Definition: encodedstream.h:48
json
static const char json[]
Definition: namespacetest.cpp:32
ReadFile
static char * ReadFile(const char *filename, Allocator &allocator)
Definition: perftest/schematest.cpp:15
EncodedStreamTest::TearDown
virtual void TearDown()
Definition: encodedstreamtest.cpp:34
EncodedStreamTest::CompareFile
bool CompareFile(const char *filename, const char *expectedFilename)
Definition: encodedstreamtest.cpp:230
EncodedStreamTest::CompareBufferFile
bool CompareBufferFile(const char *actualBuffer, size_t actualLength, const char *expectedFilename)
Definition: encodedstreamtest.cpp:240
UTF32
UTF-32 encoding.
Definition: encodings.h:418
EncodedInputStream
Input byte stream wrapper with a statically bound encoding.
Definition: encodedstream.h:39
GenericMemoryBuffer::GetBuffer
const Ch * GetBuffer() const
Definition: memorybuffer.h:50
EncodedStreamTest::Open
static FILE * Open(const char *filename)
Definition: encodedstreamtest.cpp:44
rapidjson
main RapidJSON namespace
Open
static bool Open(FileStreamType &fs, const char *filename)
Definition: istreamwrappertest.cpp:106
EncodedInputStream::Tell
size_t Tell() const
Definition: encodedstream.h:50
AutoUTFInputStream
Input stream wrapper with dynamically bound encoding and automatic encoding detection.
Definition: encodedstream.h:135
EncodedStreamTest::ReadFile
static char * ReadFile(const char *filename, bool appendPath, size_t *outLength)
Definition: encodedstreamtest.cpp:62
FileReadStream
File byte stream for input using fread().
Definition: filereadstream.h:34
AutoUTFInputStream::GetType
UTFType GetType() const
Definition: encodedstream.h:153
kUTF32LE
UTF-32 little endian.
Definition: encodings.h:607
AutoUTFInputStream::Tell
size_t Tell() const
Definition: encodedstream.h:158
EncodedStreamTest::TestAutoUTFOutputStream
void TestAutoUTFOutputStream(UTFType type, bool putBOM, const char *expectedFilename)
Definition: encodedstreamtest.cpp:196
EncodedStreamTest::SetUp
virtual void SetUp()
Definition: encodedstreamtest.cpp:30
EncodedStreamTest::TestEncodedOutputStream
void TestEncodedOutputStream(const char *expectedFilename, bool putBOM)
Definition: encodedstreamtest.cpp:163

choreo_rapidjson
Author(s):
autogenerated on Thu Jul 18 2019 03:59:09