scantoken.cpp
Go to the documentation of this file.
1 #include "scanner.h"
2 #include "token.h"
4 #include "exp.h"
5 #include "scanscalar.h"
6 #include "scantag.h"
7 #include "tag.h"
8 #include <sstream>
9 
10 namespace YAML_PM
11 {
13  // Specialization for scanning specific tokens
14 
15  // Directive
16  // . Note: no semantic checking is done here (that's for the parser to do)
18  {
20  std::vector <std::string> params;
21 
22  // pop indents and simple keys
23  PopAllIndents();
25 
26  m_simpleKeyAllowed = false;
27  m_canBeJSONFlow = false;
28 
29  // store pos and eat indicator
30  Token token(Token::DIRECTIVE, INPUT.mark());
31  INPUT.eat(1);
32 
33  // read name
34  while(INPUT && !Exp::BlankOrBreak().Matches(INPUT))
35  token.value += INPUT.get();
36 
37  // read parameters
38  while(1) {
39  // first get rid of whitespace
40  while(Exp::Blank().Matches(INPUT))
41  INPUT.eat(1);
42 
43  // break on newline or comment
45  break;
46 
47  // now read parameter
48  std::string param;
49  while(INPUT && !Exp::BlankOrBreak().Matches(INPUT))
50  param += INPUT.get();
51 
52  token.params.push_back(param);
53  }
54 
55  m_tokens.push(token);
56  }
57 
58  // DocStart
60  {
61  PopAllIndents();
63  m_simpleKeyAllowed = false;
64  m_canBeJSONFlow = false;
65 
66  // eat
67  Mark mark = INPUT.mark();
68  INPUT.eat(3);
69  m_tokens.push(Token(Token::DOC_START, mark));
70  }
71 
72  // DocEnd
74  {
75  PopAllIndents();
77  m_simpleKeyAllowed = false;
78  m_canBeJSONFlow = false;
79 
80  // eat
81  Mark mark = INPUT.mark();
82  INPUT.eat(3);
83  m_tokens.push(Token(Token::DOC_END, mark));
84  }
85 
86  // FlowStart
88  {
89  // flows can be simple keys
91  m_simpleKeyAllowed = true;
92  m_canBeJSONFlow = false;
93 
94  // eat
95  Mark mark = INPUT.mark();
96  char ch = INPUT.get();
97  FLOW_MARKER flowType = (ch == Keys::FlowSeqStart ? FLOW_SEQ : FLOW_MAP);
98  m_flows.push(flowType);
100  m_tokens.push(Token(type, mark));
101  }
102 
103  // FlowEnd
105  {
106  if(InBlockContext())
108 
109  // we might have a solo entry in the flow context
110  if(InFlowContext()) {
111  if(m_flows.top() == FLOW_MAP && VerifySimpleKey())
113  else if(m_flows.top() == FLOW_SEQ)
115  }
116 
117  m_simpleKeyAllowed = false;
118  m_canBeJSONFlow = true;
119 
120  // eat
121  Mark mark = INPUT.mark();
122  char ch = INPUT.get();
123 
124  // check that it matches the start
125  FLOW_MARKER flowType = (ch == Keys::FlowSeqEnd ? FLOW_SEQ : FLOW_MAP);
126  if(m_flows.top() != flowType)
127  throw ParserException(mark, ErrorMsg::FLOW_END);
128  m_flows.pop();
129 
131  m_tokens.push(Token(type, mark));
132  }
133 
134  // FlowEntry
136  {
137  // we might have a solo entry in the flow context
138  if(InFlowContext()) {
139  if(m_flows.top() == FLOW_MAP && VerifySimpleKey())
141  else if(m_flows.top() == FLOW_SEQ)
143  }
144 
145  m_simpleKeyAllowed = true;
146  m_canBeJSONFlow = false;
147 
148  // eat
149  Mark mark = INPUT.mark();
150  INPUT.eat(1);
151  m_tokens.push(Token(Token::FLOW_ENTRY, mark));
152  }
153 
154  // BlockEntry
156  {
157  // we better be in the block context!
158  if(InFlowContext())
160 
161  // can we put it here?
162  if(!m_simpleKeyAllowed)
164 
166  m_simpleKeyAllowed = true;
167  m_canBeJSONFlow = false;
168 
169  // eat
170  Mark mark = INPUT.mark();
171  INPUT.eat(1);
172  m_tokens.push(Token(Token::BLOCK_ENTRY, mark));
173  }
174 
175  // Key
177  {
178  // handle keys diffently in the block context (and manage indents)
179  if(InBlockContext()) {
180  if(!m_simpleKeyAllowed)
182 
184  }
185 
186  // can only put a simple key here if we're in block context
188 
189  // eat
190  Mark mark = INPUT.mark();
191  INPUT.eat(1);
192  m_tokens.push(Token(Token::KEY, mark));
193  }
194 
195  // Value
197  {
198  // and check that simple key
199  bool isSimpleKey = VerifySimpleKey();
200  m_canBeJSONFlow = false;
201 
202  if(isSimpleKey) {
203  // can't follow a simple key with another simple key (dunno why, though - it seems fine)
204  m_simpleKeyAllowed = false;
205  } else {
206  // handle values diffently in the block context (and manage indents)
207  if(InBlockContext()) {
208  if(!m_simpleKeyAllowed)
210 
212  }
213 
214  // can only put a simple key here if we're in block context
216  }
217 
218  // eat
219  Mark mark = INPUT.mark();
220  INPUT.eat(1);
221  m_tokens.push(Token(Token::VALUE, mark));
222  }
223 
224  // AnchorOrAlias
226  {
227  bool alias;
229 
230  // insert a potential simple key
232  m_simpleKeyAllowed = false;
233  m_canBeJSONFlow = false;
234 
235  // eat the indicator
236  Mark mark = INPUT.mark();
237  char indicator = INPUT.get();
238  alias = (indicator == Keys::Alias);
239 
240  // now eat the content
241  while(INPUT && Exp::Anchor().Matches(INPUT))
242  name += INPUT.get();
243 
244  // we need to have read SOMETHING!
245  if(name.empty())
247 
248  // and needs to end correctly
249  if(INPUT && !Exp::AnchorEnd().Matches(INPUT))
251 
252  // and we're done
253  Token token(alias ? Token::ALIAS : Token::ANCHOR, mark);
254  token.value = name;
255  m_tokens.push(token);
256  }
257 
258  // Tag
260  {
261  // insert a potential simple key
263  m_simpleKeyAllowed = false;
264  m_canBeJSONFlow = false;
265 
266  Token token(Token::TAG, INPUT.mark());
267 
268  // eat the indicator
269  INPUT.get();
270 
273 
274  token.value = tag;
275  token.data = Tag::VERBATIM;
276  } else {
277  bool canBeHandle;
278  token.value = ScanTagHandle(INPUT, canBeHandle);
279  if(!canBeHandle && token.value.empty())
280  token.data = Tag::NON_SPECIFIC;
281  else if(token.value.empty())
282  token.data = Tag::SECONDARY_HANDLE;
283  else
284  token.data = Tag::PRIMARY_HANDLE;
285 
286  // is there a suffix?
287  if(canBeHandle && INPUT.peek() == Keys::Tag) {
288  // eat the indicator
289  INPUT.get();
290  token.params.push_back(ScanTagSuffix(INPUT));
291  token.data = Tag::NAMED_HANDLE;
292  }
293  }
294 
295  m_tokens.push(token);
296  }
297 
298  // PlainScalar
300  {
301  std::string scalar;
302 
303  // set up the scanning parameters
306  params.eatEnd = false;
307  params.indent = (InFlowContext() ? 0 : GetTopIndent() + 1);
308  params.fold = FOLD_FLOW;
309  params.eatLeadingWhitespace = true;
310  params.trimTrailingSpaces = true;
311  params.chomp = STRIP;
312  params.onDocIndicator = BREAK;
313  params.onTabInIndentation = THROW;
314 
315  // insert a potential simple key
317 
318  Mark mark = INPUT.mark();
319  scalar = ScanScalar(INPUT, params);
320 
321  // can have a simple key only if we ended the scalar by starting a new line
323  m_canBeJSONFlow = false;
324 
325  // finally, check and see if we ended on an illegal character
326  //if(Exp::IllegalCharInScalar.Matches(INPUT))
327  // throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
328 
329  Token token(Token::PLAIN_SCALAR, mark);
330  token.value = scalar;
331  m_tokens.push(token);
332  }
333 
334  // QuotedScalar
336  {
337  std::string scalar;
338 
339  // peek at single or double quote (don't eat because we need to preserve (for the time being) the input position)
340  char quote = INPUT.peek();
341  bool single = (quote == '\'');
342 
343  // setup the scanning parameters
345  params.end = (single ? RegEx(quote) && !Exp::EscSingleQuote() : RegEx(quote));
346  params.eatEnd = true;
347  params.escape = (single ? '\'' : '\\');
348  params.indent = 0;
349  params.fold = FOLD_FLOW;
350  params.eatLeadingWhitespace = true;
351  params.trimTrailingSpaces = false;
352  params.chomp = CLIP;
353  params.onDocIndicator = THROW;
354 
355  // insert a potential simple key
357 
358  Mark mark = INPUT.mark();
359 
360  // now eat that opening quote
361  INPUT.get();
362 
363  // and scan
364  scalar = ScanScalar(INPUT, params);
365  m_simpleKeyAllowed = false;
366  m_canBeJSONFlow = true;
367 
368  Token token(Token::NON_PLAIN_SCALAR, mark);
369  token.value = scalar;
370  m_tokens.push(token);
371  }
372 
373  // BlockScalarToken
374  // . These need a little extra processing beforehand.
375  // . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
376  // and then we need to figure out what level of indentation we'll be using.
378  {
379  std::string scalar;
380 
382  params.indent = 1;
383  params.detectIndent = true;
384 
385  // eat block indicator ('|' or '>')
386  Mark mark = INPUT.mark();
387  char indicator = INPUT.get();
388  params.fold = (indicator == Keys::FoldedScalar ? FOLD_BLOCK : DONT_FOLD);
389 
390  // eat chomping/indentation indicators
391  params.chomp = CLIP;
392  int n = Exp::Chomp().Match(INPUT);
393  for(int i=0;i<n;i++) {
394  char ch = INPUT.get();
395  if(ch == '+')
396  params.chomp = KEEP;
397  else if(ch == '-')
398  params.chomp = STRIP;
399  else if(Exp::Digit().Matches(ch)) {
400  if(ch == '0')
402 
403  params.indent = ch - '0';
404  params.detectIndent = false;
405  }
406  }
407 
408  // now eat whitespace
409  while(Exp::Blank().Matches(INPUT))
410  INPUT.eat(1);
411 
412  // and comments to the end of the line
413  if(Exp::Comment().Matches(INPUT))
414  while(INPUT && !Exp::Break().Matches(INPUT))
415  INPUT.eat(1);
416 
417  // if it's not a line break, then we ran into a bad character inline
418  if(INPUT && !Exp::Break().Matches(INPUT))
420 
421  // set the initial indentation
422  if(GetTopIndent() >= 0)
423  params.indent += GetTopIndent();
424 
425  params.eatLeadingWhitespace = false;
426  params.trimTrailingSpaces = false;
427  params.onTabInIndentation = THROW;
428 
429  scalar = ScanScalar(INPUT, params);
430 
431  // simple keys always ok after block scalars (since we're gonna start a new line anyways)
432  m_simpleKeyAllowed = true;
433  m_canBeJSONFlow = false;
434 
435  Token token(Token::NON_PLAIN_SCALAR, mark);
436  token.value = scalar;
437  m_tokens.push(token);
438  }
439 }
const char FoldedScalar
Definition: exp.h:190
const RegEx & Chomp()
Definition: exp.h:169
void ScanBlockScalar()
Definition: scantoken.cpp:377
Stream INPUT
Definition: scanner.h:115
void ScanFlowStart()
Definition: scantoken.cpp:87
const char *const ALIAS_NOT_FOUND
Definition: exceptions.h:49
const char *const ANCHOR_NOT_FOUND
Definition: exceptions.h:50
std::queue< Token > m_tokens
Definition: scanner.h:118
const char *const CHAR_IN_ALIAS
Definition: exceptions.h:51
const char *const ZERO_INDENT_IN_BLOCK
Definition: exceptions.h:53
const char *const MAP_KEY
Definition: exceptions.h:47
const RegEx & Anchor()
Definition: exp.h:117
void ScanDirective()
Definition: scantoken.cpp:17
void ScanPlainScalar()
Definition: scantoken.cpp:299
const char Alias
Definition: exp.h:186
IndentMarker * PushIndentTo(int column, IndentMarker::INDENT_TYPE type)
Definition: scanner.cpp:278
std::string ScanScalar(Stream &INPUT, ScanScalarParams &params)
Definition: scanscalar.cpp:19
::std::string string
Definition: gtest.h:1979
void ScanFlowEntry()
Definition: scantoken.cpp:135
std::stack< FLOW_MARKER > m_flows
Definition: scanner.h:127
const std::string ScanVerbatimTag(Stream &INPUT)
Definition: scantag.cpp:8
bool m_simpleKeyAllowed
Definition: scanner.h:122
const RegEx & BlankOrBreak()
Definition: exp.h:38
bool InBlockContext() const
Definition: scanner.h:59
const RegEx & Digit()
Definition: exp.h:42
int GetTopIndent() const
Definition: scanner.cpp:366
const RegEx & Break()
Definition: exp.h:34
void PopAllIndents()
Definition: scanner.cpp:331
void ScanBlockEntry()
Definition: scantoken.cpp:155
const char FlowSeqEnd
Definition: exp.h:182
void InvalidateSimpleKey()
Definition: simplekey.cpp:87
void eat(int n=1)
Definition: stream.cpp:272
const char *const BLOCK_ENTRY
Definition: exceptions.h:46
const char *const CHAR_IN_ANCHOR
Definition: exceptions.h:52
const RegEx & EndScalarInFlow()
Definition: exp.h:151
int column() const
Definition: stream.h:41
void ScanQuotedScalar()
Definition: scantoken.cpp:335
const char *const FLOW_END
Definition: exceptions.h:45
bool InFlowContext() const
Definition: scanner.h:58
const std::string ScanTagHandle(Stream &INPUT, bool &canBeHandle)
Definition: scantag.cpp:32
int Match(const std::string &str) const
Definition: regeximpl.h:42
void ScanAnchorOrAlias()
Definition: scantoken.cpp:225
const RegEx & AnchorEnd()
Definition: exp.h:121
bool m_canBeJSONFlow
Definition: scanner.h:123
void InsertPotentialSimpleKey()
Definition: simplekey.cpp:60
const char *const CHAR_IN_BLOCK
Definition: exceptions.h:54
char peek() const
Definition: stream.cpp:228
const RegEx & Blank()
Definition: exp.h:30
const RegEx & EndScalar()
Definition: exp.h:147
bool VerifySimpleKey()
Definition: simplekey.cpp:104
const char Tag
Definition: exp.h:188
const char *const MAP_VALUE
Definition: exceptions.h:48
const Mark mark() const
Definition: stream.h:38
void ScanDocStart()
Definition: scantoken.cpp:59
const RegEx & EscSingleQuote()
Definition: exp.h:156
const std::string ScanTagSuffix(Stream &INPUT)
Definition: scantag.cpp:66
void PopAllSimpleKeys()
Definition: simplekey.cpp:133
const char FlowSeqStart
Definition: exp.h:181
PM::Matches Matches
const char VerbatimTagStart
Definition: exp.h:191
bool Matches(char ch) const
Definition: regeximpl.h:16
std::string value
Definition: token.h:79
const RegEx Comment()
Definition: exp.h:113


libpointmatcher
Author(s):
autogenerated on Sat May 27 2023 02:38:03