switch the core ninja parser to use re2c for the lexer

- Delete the old "Tokenizer" code.
- Write separate tests for the lexer distinct from the parser.
- Switch the parser to use the new code.
- New lexer error output has file:line numbers so e.g. Emacs can
  jump your editor to the syntax error.
- The EvalEnv ($-interpolation) code is now part of the lexer as well.

Author: evmar

configure.py

@@ -151,12 +151,13 @@ def binary(name):
     objs += cxx('browse', order_only=built('browse_py.h'))
     n.newline()
 
-n.comment('the depfile parser is generated using re2c.')
+n.comment('the depfile parser and ninja lexers are generated using re2c.')
 n.rule('re2c',
        command='re2c -b -i --no-generation-date -o $out $in',
        description='RE2C $out')
-# Generate the .cc file in the source directory so we can check it in.
+# Generate the .cc files in the source directory so we can check them in.
 n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc'))
+n.build(src('lexer.cc'), 're2c', src('lexer.in.cc'))
 n.newline()
 
 n.comment('Core source files all build into ninja library.')
@@ -169,6 +170,7 @@ def binary(name):
              'eval_env',
              'graph',
              'graphviz',
+             'lexer',
              'parsers',
              'state',
              'util']:
@@ -219,8 +221,8 @@ def binary(name):
              'depfile_parser_test',
              'disk_interface_test',
              'edit_distance_test',
-             'eval_env_test',
              'graph_test',
+             'lexer_test',
              'parsers_test',
              'state_test',
              'subprocess_test',

src/eval_env.cc

@@ -27,64 +27,6 @@ void BindingEnv::AddBinding(const string& key, const string& val) {
   bindings_[key] = val;
 }
 
-bool EvalString::Parse(const string& input, string* err, size_t* err_index) {
-  unparsed_ = input;
-
-  string::size_type start, end;
-  start = 0;
-  do {
-    end = input.find('$', start);
-    if (end == string::npos) {
-      end = input.size();
-      break;
-    }
-    if (end > start)
-      parsed_.push_back(make_pair(input.substr(start, end - start), RAW));
-    start = end + 1;
-    if (start < input.size() && input[start] == '{') {
-      ++start;
-      for (end = start + 1; end < input.size(); ++end) {
-        if (input[end] == '}')
-          break;
-      }
-      if (end >= input.size()) {
-        *err = "expected closing curly after ${";
-        if (err_index)
-          *err_index = end;
-        return false;
-      }
-      parsed_.push_back(make_pair(input.substr(start, end - start), SPECIAL));
-      ++end;
-    } else if (start < input.size() && input[start] == '$') {
-      parsed_.push_back(make_pair("$", RAW));
-      end = start + 1;
-    } else if (start < input.size() && input[start] == ' ') {
-      parsed_.push_back(make_pair(" ", RAW));
-      end = start + 1;
-    } else {
-      for (end = start; end < input.size(); ++end) {
-        char c = input[end];
-        if (!(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') ||
-              ('0' <= c && c <= '9') || c == '_')) {
-          break;
-        }
-      }
-      if (end == start) {
-        *err = "expected variable after $";
-        if (err_index)
-          *err_index = start;
-        return false;
-      }
-      parsed_.push_back(make_pair(input.substr(start, end - start), SPECIAL));
-    }
-    start = end;
-  } while (end < input.size());
-  if (end > start)
-    parsed_.push_back(make_pair(input.substr(start, end - start), RAW));
-
-  return true;
-}
-
 string EvalString::Evaluate(Env* env) const {
   string result;
   for (TokenList::const_iterator i = parsed_.begin(); i != parsed_.end(); ++i) {
@@ -95,3 +37,25 @@ string EvalString::Evaluate(Env* env) const {
   }
   return result;
 }
+
+void EvalString::Add(TokenType type, StringPiece text) {
+  // Add it to the end of an existing RAW token if possible.
+  if (type == RAW && !parsed_.empty() && parsed_.back().second == RAW) {
+    parsed_.back().first.append(text.str_, text.len_);
+  } else {
+    parsed_.push_back(make_pair(text.AsString(), type));
+  }
+}
+
+string EvalString::Serialize() const {
+  string result;
+  for (TokenList::const_iterator i = parsed_.begin();
+       i != parsed_.end(); ++i) {
+    result.append("[");
+    if (i->second == SPECIAL)
+      result.append("$");
+    result.append(i->first);
+    result.append("]");
+  }
+  return result;
+}

src/eval_env.h

@@ -20,6 +20,8 @@
 #include <vector>
 using namespace std;
 
+#include "string_piece.h"
+
 /// An interface for a scope for variable (e.g. "$foo") lookups.
 struct Env {
   virtual ~Env() {}
@@ -41,14 +43,18 @@ struct BindingEnv : public Env {
 /// A tokenized string that contains variable references.
 /// Can be evaluated relative to an Env.
 struct EvalString {
-  bool Parse(const string& input, string* err, size_t* err_index=NULL);
   string Evaluate(Env* env) const;
 
-  const string& unparsed() const { return unparsed_; }
-  bool empty() const { return unparsed_.empty(); }
+  void Clear() { parsed_.clear(); }
+  bool empty() const { return parsed_.empty(); }
 
-  string unparsed_;
   enum TokenType { RAW, SPECIAL };
+  void Add(TokenType type, StringPiece text);
+
+  /// Construct a human-readable representation of the parsed state,
+  /// for use in tests.
+  string Serialize() const;
+
   typedef vector<pair<string, TokenType> > TokenList;
   TokenList parsed_;
 };

src/eval_env_test.cc

@@ -109,10 +109,10 @@ struct Rule {
   const EvalString& description() const { return description_; }
   const EvalString& depfile() const { return depfile_; }
 
- private:
+  // TODO: private:
+
   // Allow the parsers to reach into this object and fill out its fields.
   friend class ManifestParser;
-  friend class ParserTest;
 
   string name_;