Ninja
manifest_parser.cc
Go to the documentation of this file.
1 // Copyright 2011 Google Inc. All Rights Reserved.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "manifest_parser.h"
16 
17 #include <stdio.h>
18 #include <stdlib.h>
19 #include <vector>
20 
21 #include "graph.h"
22 #include "metrics.h"
23 #include "state.h"
24 #include "util.h"
25 #include "version.h"
26 
28  : state_(state), file_reader_(file_reader) {
29  env_ = &state->bindings_;
30 }
31 
32 bool ManifestParser::Load(const string& filename, string* err, Lexer* parent) {
33  METRIC_RECORD(".ninja parse");
34  string contents;
35  string read_err;
36  if (!file_reader_->ReadFile(filename, &contents, &read_err)) {
37  *err = "loading '" + filename + "': " + read_err;
38  if (parent)
39  parent->Error(string(*err), err);
40  return false;
41  }
42 
43  // The lexer needs a nul byte at the end of its input, to know when it's done.
44  // It takes a StringPiece, and StringPiece's string constructor uses
45  // string::data(). data()'s return value isn't guaranteed to be
46  // null-terminated (although in practice - libc++, libstdc++, msvc's stl --
47  // it is, and C++11 demands that too), so add an explicit nul byte.
48  contents.resize(contents.size() + 1);
49 
50  return Parse(filename, contents, err);
51 }
52 
53 bool ManifestParser::Parse(const string& filename, const string& input,
54  string* err) {
55  lexer_.Start(filename, input);
56 
57  for (;;) {
58  Lexer::Token token = lexer_.ReadToken();
59  switch (token) {
60  case Lexer::POOL:
61  if (!ParsePool(err))
62  return false;
63  break;
64  case Lexer::BUILD:
65  if (!ParseEdge(err))
66  return false;
67  break;
68  case Lexer::RULE:
69  if (!ParseRule(err))
70  return false;
71  break;
72  case Lexer::DEFAULT:
73  if (!ParseDefault(err))
74  return false;
75  break;
76  case Lexer::IDENT: {
78  string name;
79  EvalString let_value;
80  if (!ParseLet(&name, &let_value, err))
81  return false;
82  string value = let_value.Evaluate(env_);
83  // Check ninja_required_version immediately so we can exit
84  // before encountering any syntactic surprises.
85  if (name == "ninja_required_version")
86  CheckNinjaVersion(value);
87  env_->AddBinding(name, value);
88  break;
89  }
90  case Lexer::INCLUDE:
91  if (!ParseFileInclude(false, err))
92  return false;
93  break;
94  case Lexer::SUBNINJA:
95  if (!ParseFileInclude(true, err))
96  return false;
97  break;
98  case Lexer::ERROR: {
99  return lexer_.Error(lexer_.DescribeLastError(), err);
100  }
101  case Lexer::TEOF:
102  return true;
103  case Lexer::NEWLINE:
104  break;
105  default:
106  return lexer_.Error(string("unexpected ") + Lexer::TokenName(token),
107  err);
108  }
109  }
110  return false; // not reached
111 }
112 
113 
114 bool ManifestParser::ParsePool(string* err) {
115  string name;
116  if (!lexer_.ReadIdent(&name))
117  return lexer_.Error("expected pool name", err);
118 
119  if (!ExpectToken(Lexer::NEWLINE, err))
120  return false;
121 
122  if (state_->LookupPool(name) != NULL)
123  return lexer_.Error("duplicate pool '" + name + "'", err);
124 
125  int depth = -1;
126 
127  while (lexer_.PeekToken(Lexer::INDENT)) {
128  string key;
129  EvalString value;
130  if (!ParseLet(&key, &value, err))
131  return false;
132 
133  if (key == "depth") {
134  string depth_string = value.Evaluate(env_);
135  depth = atol(depth_string.c_str());
136  if (depth < 0)
137  return lexer_.Error("invalid pool depth", err);
138  } else {
139  return lexer_.Error("unexpected variable '" + key + "'", err);
140  }
141  }
142 
143  if (depth < 0)
144  return lexer_.Error("expected 'depth =' line", err);
145 
146  state_->AddPool(new Pool(name, depth));
147  return true;
148 }
149 
150 
151 bool ManifestParser::ParseRule(string* err) {
152  string name;
153  if (!lexer_.ReadIdent(&name))
154  return lexer_.Error("expected rule name", err);
155 
156  if (!ExpectToken(Lexer::NEWLINE, err))
157  return false;
158 
159  if (state_->LookupRule(name) != NULL)
160  return lexer_.Error("duplicate rule '" + name + "'", err);
161 
162  Rule* rule = new Rule(name); // XXX scoped_ptr
163 
164  while (lexer_.PeekToken(Lexer::INDENT)) {
165  string key;
166  EvalString value;
167  if (!ParseLet(&key, &value, err))
168  return false;
169 
170  if (Rule::IsReservedBinding(key)) {
171  rule->AddBinding(key, value);
172  } else {
173  // Die on other keyvals for now; revisit if we want to add a
174  // scope here.
175  return lexer_.Error("unexpected variable '" + key + "'", err);
176  }
177  }
178 
179  if (rule->bindings_["rspfile"].empty() !=
180  rule->bindings_["rspfile_content"].empty()) {
181  return lexer_.Error("rspfile and rspfile_content need to be "
182  "both specified", err);
183  }
184 
185  if (rule->bindings_["command"].empty())
186  return lexer_.Error("expected 'command =' line", err);
187 
188  state_->AddRule(rule);
189  return true;
190 }
191 
192 bool ManifestParser::ParseLet(string* key, EvalString* value, string* err) {
193  if (!lexer_.ReadIdent(key))
194  return false;
195  if (!ExpectToken(Lexer::EQUALS, err))
196  return false;
197  if (!lexer_.ReadVarValue(value, err))
198  return false;
199  return true;
200 }
201 
202 bool ManifestParser::ParseDefault(string* err) {
203  EvalString eval;
204  if (!lexer_.ReadPath(&eval, err))
205  return false;
206  if (eval.empty())
207  return lexer_.Error("expected target name", err);
208 
209  do {
210  string path = eval.Evaluate(env_);
211  string path_err;
212  if (!CanonicalizePath(&path, &path_err))
213  return lexer_.Error(path_err, err);
214  if (!state_->AddDefault(path, &path_err))
215  return lexer_.Error(path_err, err);
216 
217  eval.Clear();
218  if (!lexer_.ReadPath(&eval, err))
219  return false;
220  } while (!eval.empty());
221 
222  if (!ExpectToken(Lexer::NEWLINE, err))
223  return false;
224 
225  return true;
226 }
227 
228 bool ManifestParser::ParseEdge(string* err) {
229  vector<EvalString> ins, outs;
230 
231  {
232  EvalString out;
233  if (!lexer_.ReadPath(&out, err))
234  return false;
235  if (out.empty())
236  return lexer_.Error("expected path", err);
237 
238  do {
239  outs.push_back(out);
240 
241  out.Clear();
242  if (!lexer_.ReadPath(&out, err))
243  return false;
244  } while (!out.empty());
245  }
246 
247  if (!ExpectToken(Lexer::COLON, err))
248  return false;
249 
250  string rule_name;
251  if (!lexer_.ReadIdent(&rule_name))
252  return lexer_.Error("expected build command name", err);
253 
254  const Rule* rule = state_->LookupRule(rule_name);
255  if (!rule)
256  return lexer_.Error("unknown build rule '" + rule_name + "'", err);
257 
258  for (;;) {
259  // XXX should we require one path here?
260  EvalString in;
261  if (!lexer_.ReadPath(&in, err))
262  return false;
263  if (in.empty())
264  break;
265  ins.push_back(in);
266  }
267 
268  // Add all implicit deps, counting how many as we go.
269  int implicit = 0;
271  for (;;) {
272  EvalString in;
273  if (!lexer_.ReadPath(&in, err))
274  return err;
275  if (in.empty())
276  break;
277  ins.push_back(in);
278  ++implicit;
279  }
280  }
281 
282  // Add all order-only deps, counting how many as we go.
283  int order_only = 0;
285  for (;;) {
286  EvalString in;
287  if (!lexer_.ReadPath(&in, err))
288  return false;
289  if (in.empty())
290  break;
291  ins.push_back(in);
292  ++order_only;
293  }
294  }
295 
296  if (!ExpectToken(Lexer::NEWLINE, err))
297  return false;
298 
299  // Bindings on edges are rare, so allocate per-edge envs only when needed.
300  bool hasIdent = lexer_.PeekToken(Lexer::INDENT);
301  BindingEnv* env = hasIdent ? new BindingEnv(env_) : env_;
302  while (hasIdent) {
303  string key;
304  EvalString val;
305  if (!ParseLet(&key, &val, err))
306  return false;
307 
308  env->AddBinding(key, val.Evaluate(env_));
309  hasIdent = lexer_.PeekToken(Lexer::INDENT);
310  }
311 
312  Edge* edge = state_->AddEdge(rule);
313  edge->env_ = env;
314 
315  string pool_name = edge->GetBinding("pool");
316  if (!pool_name.empty()) {
317  Pool* pool = state_->LookupPool(pool_name);
318  if (pool == NULL)
319  return lexer_.Error("unknown pool name '" + pool_name + "'", err);
320  edge->pool_ = pool;
321  }
322 
323  for (vector<EvalString>::iterator i = ins.begin(); i != ins.end(); ++i) {
324  string path = i->Evaluate(env);
325  string path_err;
326  if (!CanonicalizePath(&path, &path_err))
327  return lexer_.Error(path_err, err);
328  state_->AddIn(edge, path);
329  }
330  for (vector<EvalString>::iterator i = outs.begin(); i != outs.end(); ++i) {
331  string path = i->Evaluate(env);
332  string path_err;
333  if (!CanonicalizePath(&path, &path_err))
334  return lexer_.Error(path_err, err);
335  state_->AddOut(edge, path);
336  }
337  edge->implicit_deps_ = implicit;
338  edge->order_only_deps_ = order_only;
339 
340  // Multiple outputs aren't (yet?) supported with depslog.
341  string deps_type = edge->GetBinding("deps");
342  if (!deps_type.empty() && edge->outputs_.size() > 1) {
343  return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; "
344  "bring this up on the mailing list if it affects you",
345  err);
346  }
347 
348  return true;
349 }
350 
351 bool ManifestParser::ParseFileInclude(bool new_scope, string* err) {
352  EvalString eval;
353  if (!lexer_.ReadPath(&eval, err))
354  return false;
355  string path = eval.Evaluate(env_);
356 
357  ManifestParser subparser(state_, file_reader_);
358  if (new_scope) {
359  subparser.env_ = new BindingEnv(env_);
360  } else {
361  subparser.env_ = env_;
362  }
363 
364  if (!subparser.Load(path, err, &lexer_))
365  return false;
366 
367  if (!ExpectToken(Lexer::NEWLINE, err))
368  return false;
369 
370  return true;
371 }
372 
373 bool ManifestParser::ExpectToken(Lexer::Token expected, string* err) {
374  Lexer::Token token = lexer_.ReadToken();
375  if (token != expected) {
376  string message = string("expected ") + Lexer::TokenName(expected);
377  message += string(", got ") + Lexer::TokenName(token);
378  message += Lexer::TokenErrorHint(expected);
379  return lexer_.Error(message, err);
380  }
381  return true;
382 }
Definition: lexer.h:27
void AddIn(Edge *edge, StringPiece path)
Definition: state.cc:148
static bool IsReservedBinding(const string &var)
Definition: graph.cc:48
bool Load(const string &filename, string *err, Lexer *parent=NULL)
Load and parse a file.
virtual bool ReadFile(const string &path, string *content, string *err)=0
int order_only_deps_
Definition: graph.h:182
int implicit_deps_
Definition: graph.h:181
BindingEnv * env_
bool ParsePool(string *err)
Parse various statement types.
void Clear()
Definition: eval_env.h:62
Pool * LookupPool(const string &pool_name)
Definition: state.cc:98
void UnreadToken()
Rewind to the last read Token.
Definition: lexer.cc:115
static const char * TokenErrorHint(Token expected)
Return a human-readable token hint, used in error messages.
Definition: lexer.cc:94
bool PeekToken(Token token)
If the next token is token, read it and return true.
Definition: lexer.cc:419
An edge in the dependency graph; links between Nodes using Rules.
Definition: graph.h:137
bool Error(const string &message, string *err)
Construct an error message with context.
Definition: lexer.cc:23
void CheckNinjaVersion(const string &version)
Check whether version is compatible with the current Ninja version, aborting if not.
Definition: version.cc:34
bool CanonicalizePath(string *path, string *err)
Canonicalize a path like "foo/../bar.h" into just "bar.h".
Definition: util.cc:88
Token ReadToken()
Read a Token from the Token enum.
Definition: lexer.cc:119
bool ReadPath(EvalString *path, string *err)
Read a path (complete with $escapes).
Definition: lexer.h:79
Edge * AddEdge(const Rule *rule)
Definition: state.cc:105
Parses .ninja files.
An Env which contains a mapping of variables to values as well as a pointer to a parent scope...
Definition: eval_env.h:35
void AddOut(Edge *edge, StringPiece path)
Definition: state.cc:154
bool ParseEdge(string *err)
void AddRule(const Rule *rule)
Definition: state.cc:81
BindingEnv * env_
Definition: graph.h:164
An invokable build command and associated metadata (description, etc.).
Definition: graph.h:116
string DescribeLastError()
If the last token read was an ERROR token, provide more info or the empty string. ...
Definition: lexer.cc:103
A pool for delayed edges.
Definition: state.h:39
bool empty() const
Definition: eval_env.h:63
void AddPool(Pool *pool)
Definition: state.cc:93
Token
Definition: lexer.h:32
bool ParseLet(string *key, EvalString *val, string *err)
#define METRIC_RECORD(name)
The primary interface to metrics.
Definition: metrics.h:85
BindingEnv bindings_
Definition: state.h:131
bool ReadIdent(string *out)
Read a simple identifier (a rule or variable name).
Definition: lexer.cc:508
Pool * pool_
Definition: graph.h:161
void AddBinding(const string &key, const EvalString &val)
Definition: graph.cc:36
bool ExpectToken(Lexer::Token expected, string *err)
If the next token is not expected, produce an error string saying "expectd foo, got bar"...
bool ReadVarValue(EvalString *value, string *err)
Read the value side of a var = value line (complete with $escapes).
Definition: lexer.h:85
bool AddDefault(StringPiece path, string *error)
Definition: state.cc:166
string GetBinding(const string &key)
Returns the shell-escaped value of |key|.
Definition: graph.cc:283
bool ParseRule(string *err)
void AddBinding(const string &key, const string &val)
Definition: eval_env.cc:26
static const char * TokenName(Token t)
Return a human-readable form of a token, used in error messages.
Definition: lexer.cc:73
Global state (file status, loaded rules) for a single run.
Definition: state.h:83
map< string, EvalString > bindings_
Definition: graph.h:133
FileReader * file_reader_
string Evaluate(Env *env) const
Definition: eval_env.cc:46
bool ParseFileInclude(bool new_scope, string *err)
Parse either a 'subninja' or 'include' line.
A tokenized string that contains variable references.
Definition: eval_env.h:59
bool Parse(const string &filename, const string &input, string *err)
Parse a file, given its contents as a string.
bool ParseDefault(string *err)
const Rule * LookupRule(const string &rule_name)
Definition: state.cc:86
void Start(StringPiece filename, StringPiece input)
Start parsing some input.
Definition: lexer.cc:66
ManifestParser(State *state, FileReader *file_reader)
vector< Node * > outputs_
Definition: graph.h:163