Ninja
lexer.cc
Go to the documentation of this file.
1 /* Generated by re2c 0.16 */
2 // Copyright 2011 Google Inc. All Rights Reserved.
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 
16 #include "lexer.h"
17 
18 #include <stdio.h>
19 
20 #include "eval_env.h"
21 #include "util.h"
22 
23 bool Lexer::Error(const string& message, string* err) {
24  // Compute line/column.
25  int line = 1;
26  const char* context = input_.str_;
27  for (const char* p = input_.str_; p < last_token_; ++p) {
28  if (*p == '\n') {
29  ++line;
30  context = p + 1;
31  }
32  }
33  int col = last_token_ ? (int)(last_token_ - context) : 0;
34 
35  char buf[1024];
36  snprintf(buf, sizeof(buf), "%s:%d: ", filename_.AsString().c_str(), line);
37  *err = buf;
38  *err += message + "\n";
39 
40  // Add some context to the message.
41  const int kTruncateColumn = 72;
42  if (col > 0 && col < kTruncateColumn) {
43  int len;
44  bool truncated = true;
45  for (len = 0; len < kTruncateColumn; ++len) {
46  if (context[len] == 0 || context[len] == '\n') {
47  truncated = false;
48  break;
49  }
50  }
51  *err += string(context, len);
52  if (truncated)
53  *err += "...";
54  *err += "\n";
55  *err += string(col, ' ');
56  *err += "^ near here";
57  }
58 
59  return false;
60 }
61 
62 Lexer::Lexer(const char* input) {
63  Start("input", input);
64 }
65 
66 void Lexer::Start(StringPiece filename, StringPiece input) {
67  filename_ = filename;
68  input_ = input;
69  ofs_ = input_.str_;
70  last_token_ = NULL;
71 }
72 
73 const char* Lexer::TokenName(Token t) {
74  switch (t) {
75  case ERROR: return "lexing error";
76  case BUILD: return "'build'";
77  case COLON: return "':'";
78  case DEFAULT: return "'default'";
79  case EQUALS: return "'='";
80  case IDENT: return "identifier";
81  case INCLUDE: return "'include'";
82  case INDENT: return "indent";
83  case NEWLINE: return "newline";
84  case PIPE2: return "'||'";
85  case PIPE: return "'|'";
86  case POOL: return "'pool'";
87  case RULE: return "'rule'";
88  case SUBNINJA: return "'subninja'";
89  case TEOF: return "eof";
90  }
91  return NULL; // not reached
92 }
93 
94 const char* Lexer::TokenErrorHint(Token expected) {
95  switch (expected) {
96  case COLON:
97  return " ($ also escapes ':')";
98  default:
99  return "";
100  }
101 }
102 
104  if (last_token_) {
105  switch (last_token_[0]) {
106  case '\r':
107  return "carriage returns are not allowed, use newlines";
108  case '\t':
109  return "tabs are not allowed, use spaces";
110  }
111  }
112  return "lexing error";
113 }
114 
116  ofs_ = last_token_;
117 }
118 
120  const char* p = ofs_;
121  const char* q;
122  const char* start;
123  Lexer::Token token;
124  for (;;) {
125  start = p;
126 
127 {
128  unsigned char yych;
129  unsigned int yyaccept = 0;
130  static const unsigned char yybm[] = {
131  0, 128, 128, 128, 128, 128, 128, 128,
132  128, 128, 0, 128, 128, 0, 128, 128,
133  128, 128, 128, 128, 128, 128, 128, 128,
134  128, 128, 128, 128, 128, 128, 128, 128,
135  160, 128, 128, 128, 128, 128, 128, 128,
136  128, 128, 128, 128, 128, 192, 192, 128,
137  192, 192, 192, 192, 192, 192, 192, 192,
138  192, 192, 128, 128, 128, 128, 128, 128,
139  128, 192, 192, 192, 192, 192, 192, 192,
140  192, 192, 192, 192, 192, 192, 192, 192,
141  192, 192, 192, 192, 192, 192, 192, 192,
142  192, 192, 192, 128, 128, 128, 128, 192,
143  128, 192, 192, 192, 192, 192, 192, 192,
144  192, 192, 192, 192, 192, 192, 192, 192,
145  192, 192, 192, 192, 192, 192, 192, 192,
146  192, 192, 192, 128, 128, 128, 128, 128,
147  128, 128, 128, 128, 128, 128, 128, 128,
148  128, 128, 128, 128, 128, 128, 128, 128,
149  128, 128, 128, 128, 128, 128, 128, 128,
150  128, 128, 128, 128, 128, 128, 128, 128,
151  128, 128, 128, 128, 128, 128, 128, 128,
152  128, 128, 128, 128, 128, 128, 128, 128,
153  128, 128, 128, 128, 128, 128, 128, 128,
154  128, 128, 128, 128, 128, 128, 128, 128,
155  128, 128, 128, 128, 128, 128, 128, 128,
156  128, 128, 128, 128, 128, 128, 128, 128,
157  128, 128, 128, 128, 128, 128, 128, 128,
158  128, 128, 128, 128, 128, 128, 128, 128,
159  128, 128, 128, 128, 128, 128, 128, 128,
160  128, 128, 128, 128, 128, 128, 128, 128,
161  128, 128, 128, 128, 128, 128, 128, 128,
162  128, 128, 128, 128, 128, 128, 128, 128,
163  };
164  yych = *p;
165  if (yybm[0+yych] & 32) {
166  goto yy8;
167  }
168  if (yych <= '_') {
169  if (yych <= '/') {
170  if (yych <= '"') {
171  if (yych <= 0x00) goto yy2;
172  if (yych == '\n') goto yy6;
173  goto yy4;
174  } else {
175  if (yych <= '#') goto yy11;
176  if (yych <= ',') goto yy4;
177  if (yych <= '.') goto yy12;
178  goto yy4;
179  }
180  } else {
181  if (yych <= '=') {
182  if (yych <= '9') goto yy12;
183  if (yych <= ':') goto yy15;
184  if (yych <= '<') goto yy4;
185  goto yy17;
186  } else {
187  if (yych <= '@') goto yy4;
188  if (yych <= 'Z') goto yy12;
189  if (yych <= '^') goto yy4;
190  goto yy12;
191  }
192  }
193  } else {
194  if (yych <= 'o') {
195  if (yych <= 'c') {
196  if (yych <= '`') goto yy4;
197  if (yych == 'b') goto yy19;
198  goto yy12;
199  } else {
200  if (yych <= 'd') goto yy20;
201  if (yych == 'i') goto yy21;
202  goto yy12;
203  }
204  } else {
205  if (yych <= 's') {
206  if (yych <= 'p') goto yy22;
207  if (yych <= 'q') goto yy12;
208  if (yych <= 'r') goto yy23;
209  goto yy24;
210  } else {
211  if (yych <= 'z') goto yy12;
212  if (yych == '|') goto yy25;
213  goto yy4;
214  }
215  }
216  }
217 yy2:
218  ++p;
219  { token = TEOF; break; }
220 yy4:
221  ++p;
222 yy5:
223  { token = ERROR; break; }
224 yy6:
225  ++p;
226  { token = NEWLINE; break; }
227 yy8:
228  yyaccept = 0;
229  q = ++p;
230  yych = *p;
231  if (yybm[0+yych] & 32) {
232  goto yy8;
233  }
234  if (yych == '\n') goto yy6;
235  if (yych == '#') goto yy27;
236 yy10:
237  { token = INDENT; break; }
238 yy11:
239  yyaccept = 1;
240  yych = *(q = ++p);
241  if (yych <= 0x00) goto yy5;
242  if (yych == '\r') goto yy5;
243  goto yy28;
244 yy12:
245  ++p;
246  yych = *p;
247 yy13:
248  if (yybm[0+yych] & 64) {
249  goto yy12;
250  }
251  { token = IDENT; break; }
252 yy15:
253  ++p;
254  { token = COLON; break; }
255 yy17:
256  ++p;
257  { token = EQUALS; break; }
258 yy19:
259  yych = *++p;
260  if (yych == 'u') goto yy32;
261  goto yy13;
262 yy20:
263  yych = *++p;
264  if (yych == 'e') goto yy33;
265  goto yy13;
266 yy21:
267  yych = *++p;
268  if (yych == 'n') goto yy34;
269  goto yy13;
270 yy22:
271  yych = *++p;
272  if (yych == 'o') goto yy35;
273  goto yy13;
274 yy23:
275  yych = *++p;
276  if (yych == 'u') goto yy36;
277  goto yy13;
278 yy24:
279  yych = *++p;
280  if (yych == 'u') goto yy37;
281  goto yy13;
282 yy25:
283  ++p;
284  if ((yych = *p) == '|') goto yy38;
285  { token = PIPE; break; }
286 yy27:
287  ++p;
288  yych = *p;
289 yy28:
290  if (yybm[0+yych] & 128) {
291  goto yy27;
292  }
293  if (yych <= 0x00) goto yy29;
294  if (yych <= '\n') goto yy30;
295 yy29:
296  p = q;
297  if (yyaccept == 0) {
298  goto yy10;
299  } else {
300  goto yy5;
301  }
302 yy30:
303  ++p;
304  { continue; }
305 yy32:
306  yych = *++p;
307  if (yych == 'i') goto yy40;
308  goto yy13;
309 yy33:
310  yych = *++p;
311  if (yych == 'f') goto yy41;
312  goto yy13;
313 yy34:
314  yych = *++p;
315  if (yych == 'c') goto yy42;
316  goto yy13;
317 yy35:
318  yych = *++p;
319  if (yych == 'o') goto yy43;
320  goto yy13;
321 yy36:
322  yych = *++p;
323  if (yych == 'l') goto yy44;
324  goto yy13;
325 yy37:
326  yych = *++p;
327  if (yych == 'b') goto yy45;
328  goto yy13;
329 yy38:
330  ++p;
331  { token = PIPE2; break; }
332 yy40:
333  yych = *++p;
334  if (yych == 'l') goto yy46;
335  goto yy13;
336 yy41:
337  yych = *++p;
338  if (yych == 'a') goto yy47;
339  goto yy13;
340 yy42:
341  yych = *++p;
342  if (yych == 'l') goto yy48;
343  goto yy13;
344 yy43:
345  yych = *++p;
346  if (yych == 'l') goto yy49;
347  goto yy13;
348 yy44:
349  yych = *++p;
350  if (yych == 'e') goto yy51;
351  goto yy13;
352 yy45:
353  yych = *++p;
354  if (yych == 'n') goto yy53;
355  goto yy13;
356 yy46:
357  yych = *++p;
358  if (yych == 'd') goto yy54;
359  goto yy13;
360 yy47:
361  yych = *++p;
362  if (yych == 'u') goto yy56;
363  goto yy13;
364 yy48:
365  yych = *++p;
366  if (yych == 'u') goto yy57;
367  goto yy13;
368 yy49:
369  ++p;
370  if (yybm[0+(yych = *p)] & 64) {
371  goto yy12;
372  }
373  { token = POOL; break; }
374 yy51:
375  ++p;
376  if (yybm[0+(yych = *p)] & 64) {
377  goto yy12;
378  }
379  { token = RULE; break; }
380 yy53:
381  yych = *++p;
382  if (yych == 'i') goto yy58;
383  goto yy13;
384 yy54:
385  ++p;
386  if (yybm[0+(yych = *p)] & 64) {
387  goto yy12;
388  }
389  { token = BUILD; break; }
390 yy56:
391  yych = *++p;
392  if (yych == 'l') goto yy59;
393  goto yy13;
394 yy57:
395  yych = *++p;
396  if (yych == 'd') goto yy60;
397  goto yy13;
398 yy58:
399  yych = *++p;
400  if (yych == 'n') goto yy61;
401  goto yy13;
402 yy59:
403  yych = *++p;
404  if (yych == 't') goto yy62;
405  goto yy13;
406 yy60:
407  yych = *++p;
408  if (yych == 'e') goto yy64;
409  goto yy13;
410 yy61:
411  yych = *++p;
412  if (yych == 'j') goto yy66;
413  goto yy13;
414 yy62:
415  ++p;
416  if (yybm[0+(yych = *p)] & 64) {
417  goto yy12;
418  }
419  { token = DEFAULT; break; }
420 yy64:
421  ++p;
422  if (yybm[0+(yych = *p)] & 64) {
423  goto yy12;
424  }
425  { token = INCLUDE; break; }
426 yy66:
427  yych = *++p;
428  if (yych != 'a') goto yy13;
429  ++p;
430  if (yybm[0+(yych = *p)] & 64) {
431  goto yy12;
432  }
433  { token = SUBNINJA; break; }
434 }
435 
436  }
437 
438  last_token_ = start;
439  ofs_ = p;
440  if (token != NEWLINE && token != TEOF)
441  EatWhitespace();
442  return token;
443 }
444 
445 bool Lexer::PeekToken(Token token) {
446  Token t = ReadToken();
447  if (t == token)
448  return true;
449  UnreadToken();
450  return false;
451 }
452 
454  const char* p = ofs_;
455  for (;;) {
456  ofs_ = p;
457 
458 {
459  unsigned char yych;
460  static const unsigned char yybm[] = {
461  0, 0, 0, 0, 0, 0, 0, 0,
462  0, 0, 0, 0, 0, 0, 0, 0,
463  0, 0, 0, 0, 0, 0, 0, 0,
464  0, 0, 0, 0, 0, 0, 0, 0,
465  128, 0, 0, 0, 0, 0, 0, 0,
466  0, 0, 0, 0, 0, 0, 0, 0,
467  0, 0, 0, 0, 0, 0, 0, 0,
468  0, 0, 0, 0, 0, 0, 0, 0,
469  0, 0, 0, 0, 0, 0, 0, 0,
470  0, 0, 0, 0, 0, 0, 0, 0,
471  0, 0, 0, 0, 0, 0, 0, 0,
472  0, 0, 0, 0, 0, 0, 0, 0,
473  0, 0, 0, 0, 0, 0, 0, 0,
474  0, 0, 0, 0, 0, 0, 0, 0,
475  0, 0, 0, 0, 0, 0, 0, 0,
476  0, 0, 0, 0, 0, 0, 0, 0,
477  0, 0, 0, 0, 0, 0, 0, 0,
478  0, 0, 0, 0, 0, 0, 0, 0,
479  0, 0, 0, 0, 0, 0, 0, 0,
480  0, 0, 0, 0, 0, 0, 0, 0,
481  0, 0, 0, 0, 0, 0, 0, 0,
482  0, 0, 0, 0, 0, 0, 0, 0,
483  0, 0, 0, 0, 0, 0, 0, 0,
484  0, 0, 0, 0, 0, 0, 0, 0,
485  0, 0, 0, 0, 0, 0, 0, 0,
486  0, 0, 0, 0, 0, 0, 0, 0,
487  0, 0, 0, 0, 0, 0, 0, 0,
488  0, 0, 0, 0, 0, 0, 0, 0,
489  0, 0, 0, 0, 0, 0, 0, 0,
490  0, 0, 0, 0, 0, 0, 0, 0,
491  0, 0, 0, 0, 0, 0, 0, 0,
492  0, 0, 0, 0, 0, 0, 0, 0,
493  };
494  yych = *p;
495  if (yybm[0+yych] & 128) {
496  goto yy75;
497  }
498  if (yych <= 0x00) goto yy71;
499  if (yych == '$') goto yy78;
500  goto yy73;
501 yy71:
502  ++p;
503  { break; }
504 yy73:
505  ++p;
506 yy74:
507  { break; }
508 yy75:
509  ++p;
510  yych = *p;
511  if (yybm[0+yych] & 128) {
512  goto yy75;
513  }
514  { continue; }
515 yy78:
516  yych = *++p;
517  if (yych != '\n') goto yy74;
518  ++p;
519  { continue; }
520 }
521 
522  }
523 }
524 
525 bool Lexer::ReadIdent(string* out) {
526  const char* p = ofs_;
527  for (;;) {
528  const char* start = p;
529 
530 {
531  unsigned char yych;
532  static const unsigned char yybm[] = {
533  0, 0, 0, 0, 0, 0, 0, 0,
534  0, 0, 0, 0, 0, 0, 0, 0,
535  0, 0, 0, 0, 0, 0, 0, 0,
536  0, 0, 0, 0, 0, 0, 0, 0,
537  0, 0, 0, 0, 0, 0, 0, 0,
538  0, 0, 0, 0, 0, 128, 128, 0,
539  128, 128, 128, 128, 128, 128, 128, 128,
540  128, 128, 0, 0, 0, 0, 0, 0,
541  0, 128, 128, 128, 128, 128, 128, 128,
542  128, 128, 128, 128, 128, 128, 128, 128,
543  128, 128, 128, 128, 128, 128, 128, 128,
544  128, 128, 128, 0, 0, 0, 0, 128,
545  0, 128, 128, 128, 128, 128, 128, 128,
546  128, 128, 128, 128, 128, 128, 128, 128,
547  128, 128, 128, 128, 128, 128, 128, 128,
548  128, 128, 128, 0, 0, 0, 0, 0,
549  0, 0, 0, 0, 0, 0, 0, 0,
550  0, 0, 0, 0, 0, 0, 0, 0,
551  0, 0, 0, 0, 0, 0, 0, 0,
552  0, 0, 0, 0, 0, 0, 0, 0,
553  0, 0, 0, 0, 0, 0, 0, 0,
554  0, 0, 0, 0, 0, 0, 0, 0,
555  0, 0, 0, 0, 0, 0, 0, 0,
556  0, 0, 0, 0, 0, 0, 0, 0,
557  0, 0, 0, 0, 0, 0, 0, 0,
558  0, 0, 0, 0, 0, 0, 0, 0,
559  0, 0, 0, 0, 0, 0, 0, 0,
560  0, 0, 0, 0, 0, 0, 0, 0,
561  0, 0, 0, 0, 0, 0, 0, 0,
562  0, 0, 0, 0, 0, 0, 0, 0,
563  0, 0, 0, 0, 0, 0, 0, 0,
564  0, 0, 0, 0, 0, 0, 0, 0,
565  };
566  yych = *p;
567  if (yybm[0+yych] & 128) {
568  goto yy85;
569  }
570  ++p;
571  { return false; }
572 yy85:
573  ++p;
574  yych = *p;
575  if (yybm[0+yych] & 128) {
576  goto yy85;
577  }
578  {
579  out->assign(start, p - start);
580  break;
581  }
582 }
583 
584  }
585  ofs_ = p;
586  EatWhitespace();
587  return true;
588 }
589 
590 bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) {
591  const char* p = ofs_;
592  const char* q;
593  const char* start;
594  for (;;) {
595  start = p;
596 
597 {
598  unsigned char yych;
599  static const unsigned char yybm[] = {
600  0, 16, 16, 16, 16, 16, 16, 16,
601  16, 16, 0, 16, 16, 0, 16, 16,
602  16, 16, 16, 16, 16, 16, 16, 16,
603  16, 16, 16, 16, 16, 16, 16, 16,
604  32, 16, 16, 16, 0, 16, 16, 16,
605  16, 16, 16, 16, 16, 208, 144, 16,
606  208, 208, 208, 208, 208, 208, 208, 208,
607  208, 208, 0, 16, 16, 16, 16, 16,
608  16, 208, 208, 208, 208, 208, 208, 208,
609  208, 208, 208, 208, 208, 208, 208, 208,
610  208, 208, 208, 208, 208, 208, 208, 208,
611  208, 208, 208, 16, 16, 16, 16, 208,
612  16, 208, 208, 208, 208, 208, 208, 208,
613  208, 208, 208, 208, 208, 208, 208, 208,
614  208, 208, 208, 208, 208, 208, 208, 208,
615  208, 208, 208, 16, 0, 16, 16, 16,
616  16, 16, 16, 16, 16, 16, 16, 16,
617  16, 16, 16, 16, 16, 16, 16, 16,
618  16, 16, 16, 16, 16, 16, 16, 16,
619  16, 16, 16, 16, 16, 16, 16, 16,
620  16, 16, 16, 16, 16, 16, 16, 16,
621  16, 16, 16, 16, 16, 16, 16, 16,
622  16, 16, 16, 16, 16, 16, 16, 16,
623  16, 16, 16, 16, 16, 16, 16, 16,
624  16, 16, 16, 16, 16, 16, 16, 16,
625  16, 16, 16, 16, 16, 16, 16, 16,
626  16, 16, 16, 16, 16, 16, 16, 16,
627  16, 16, 16, 16, 16, 16, 16, 16,
628  16, 16, 16, 16, 16, 16, 16, 16,
629  16, 16, 16, 16, 16, 16, 16, 16,
630  16, 16, 16, 16, 16, 16, 16, 16,
631  16, 16, 16, 16, 16, 16, 16, 16,
632  };
633  yych = *p;
634  if (yybm[0+yych] & 16) {
635  goto yy92;
636  }
637  if (yych <= '\r') {
638  if (yych <= 0x00) goto yy90;
639  if (yych <= '\n') goto yy95;
640  goto yy97;
641  } else {
642  if (yych <= ' ') goto yy95;
643  if (yych <= '$') goto yy99;
644  goto yy95;
645  }
646 yy90:
647  ++p;
648  {
649  last_token_ = start;
650  return Error("unexpected EOF", err);
651  }
652 yy92:
653  ++p;
654  yych = *p;
655  if (yybm[0+yych] & 16) {
656  goto yy92;
657  }
658  {
659  eval->AddText(StringPiece(start, p - start));
660  continue;
661  }
662 yy95:
663  ++p;
664  {
665  if (path) {
666  p = start;
667  break;
668  } else {
669  if (*start == '\n')
670  break;
671  eval->AddText(StringPiece(start, 1));
672  continue;
673  }
674  }
675 yy97:
676  ++p;
677  {
678  last_token_ = start;
679  return Error(DescribeLastError(), err);
680  }
681 yy99:
682  yych = *++p;
683  if (yybm[0+yych] & 64) {
684  goto yy109;
685  }
686  if (yych <= '#') {
687  if (yych <= '\n') {
688  if (yych >= '\n') goto yy102;
689  } else {
690  if (yych == ' ') goto yy105;
691  }
692  } else {
693  if (yych <= ':') {
694  if (yych <= '$') goto yy107;
695  if (yych >= '0') goto yy112;
696  } else {
697  if (yych <= '`') goto yy100;
698  if (yych <= '{') goto yy114;
699  }
700  }
701 yy100:
702  ++p;
703 yy101:
704  {
705  last_token_ = start;
706  return Error("bad $-escape (literal $ must be written as $$)", err);
707  }
708 yy102:
709  ++p;
710  yych = *p;
711  if (yybm[0+yych] & 32) {
712  goto yy102;
713  }
714  {
715  continue;
716  }
717 yy105:
718  ++p;
719  {
720  eval->AddText(StringPiece(" ", 1));
721  continue;
722  }
723 yy107:
724  ++p;
725  {
726  eval->AddText(StringPiece("$", 1));
727  continue;
728  }
729 yy109:
730  ++p;
731  yych = *p;
732  if (yybm[0+yych] & 64) {
733  goto yy109;
734  }
735  {
736  eval->AddSpecial(StringPiece(start + 1, p - start - 1));
737  continue;
738  }
739 yy112:
740  ++p;
741  {
742  eval->AddText(StringPiece(":", 1));
743  continue;
744  }
745 yy114:
746  yych = *(q = ++p);
747  if (yybm[0+yych] & 128) {
748  goto yy115;
749  }
750  goto yy101;
751 yy115:
752  ++p;
753  yych = *p;
754  if (yybm[0+yych] & 128) {
755  goto yy115;
756  }
757  if (yych == '}') goto yy118;
758  p = q;
759  goto yy101;
760 yy118:
761  ++p;
762  {
763  eval->AddSpecial(StringPiece(start + 2, p - start - 3));
764  continue;
765  }
766 }
767 
768  }
769  last_token_ = start;
770  ofs_ = p;
771  if (path)
772  EatWhitespace();
773  // Non-path strings end in newlines, so there's no whitespace to eat.
774  return true;
775 }
const char * last_token_
Definition: lexer.h:102
const char * str_
Definition: string_piece.h:49
void UnreadToken()
Rewind to the last read Token.
Definition: lexer.cc:115
StringPiece represents a slice of a string whose memory is managed externally.
Definition: string_piece.h:27
static const char * TokenErrorHint(Token expected)
Return a human-readable token hint, used in error messages.
Definition: lexer.cc:94
void EatWhitespace()
Skip past whitespace (called after each read token/ident/etc.).
Definition: lexer.cc:453
string AsString() const
Convert the slice into a full-fledged std::string, copying the data into a new string.
Definition: string_piece.h:45
bool PeekToken(Token token)
If the next token is token, read it and return true.
Definition: lexer.cc:445
bool Error(const string &message, string *err)
Construct an error message with context.
Definition: lexer.cc:23
StringPiece filename_
Definition: lexer.h:99
StringPiece input_
Definition: lexer.h:100
Token ReadToken()
Read a Token from the Token enum.
Definition: lexer.cc:119
void AddSpecial(StringPiece text)
Definition: eval_env.cc:65
string DescribeLastError()
If the last token read was an ERROR token, provide more info or the empty string. ...
Definition: lexer.cc:103
Token
Definition: lexer.h:32
bool ReadIdent(string *out)
Read a simple identifier (a rule or variable name).
Definition: lexer.cc:525
bool ReadEvalString(EvalString *eval, bool path, string *err)
Read a $-escaped string.
Definition: lexer.cc:590
void AddText(StringPiece text)
Definition: eval_env.cc:57
static const char * TokenName(Token t)
Return a human-readable form of a token, used in error messages.
Definition: lexer.cc:73
Lexer()
Definition: lexer.h:28
A tokenized string that contains variable references.
Definition: eval_env.h:59
const char * ofs_
Definition: lexer.h:101
void Start(StringPiece filename, StringPiece input)
Start parsing some input.
Definition: lexer.cc:66