swift-syntax-parser-test.cpp

//===--- swift-syntax-parser-test.cpp - Test util for C parser library ----===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// Testing utility for the C API of the parser library.
//
//===----------------------------------------------------------------------===//

#include "swift-c/SyntaxParser/SwiftSyntaxParser.h"
#include "swift/Basic/LLVM.h"
#include "swift/Basic/LLVMInitialize.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/SourceMgr.h"
#include "llvm/Support/Timer.h"

using namespace swift;
using namespace llvm;

enum class ActionType {
  DumpTree,
  Time,
  Diagnostics,
};

namespace options {

static cl::opt<ActionType>
Action(cl::desc("Action (required):"),
       cl::init(ActionType::DumpTree),
       cl::values(
        clEnumValN(ActionType::DumpTree,
                   "dump-tree",
                   "Parse the source file and dump syntax tree"),
        clEnumValN(ActionType::Diagnostics,
                   "dump-diags",
                   "Parse the source file and dump parser diagnostics"),
        clEnumValN(ActionType::Time,
                   "time",
                   "Time parsing, use '-n' to specify number of invocations")));

static cl::list<std::string>
Filename(cl::Positional, cl::desc("source file"), cl::Required);

static cl::opt<unsigned>
NumParses("n", cl::desc("number of invocations"), cl::init(1));

static cl::opt<std::string>
    SwiftVersion("swift-version",
                 cl::desc("Interpret input according to a specific Swift "
                          "language version number"));

static cl::opt<bool>
    EnableBareSlashRegex("enable-bare-slash-regex",
                         cl::desc("Enable or disable the use of forward slash "
                                  "regular-expression literal syntax"));
}

namespace {
struct SPNode {
  swiftparse_syntax_kind_t kind;

  Optional<swiftparse_token_kind_t> tokKind;
  StringRef leadingTriviaText;
  StringRef tokenText;
  StringRef trailingTriviaText;

  std::vector<std::unique_ptr<SPNode>> members;

  LLVM_DUMP_METHOD void dump() {
    dump(errs());
  }

  void dump(raw_ostream &OS) {
    // FIXME: Return the syntax/token kinds directly from the C API, instead
    // of the serialization number, and print the kind identifier for each,
    // instead of the number.
    if (tokKind.hasValue()) {
      OS << "<t" << (unsigned)tokKind.getValue() << '>';
      OS << leadingTriviaText << '|' << tokenText << '|' << trailingTriviaText;
      OS << "</t" << (unsigned)tokKind.getValue() << '>';
    } else {
      OS << "<s" << (unsigned)kind << '>';
      for (const auto &mn : members) {
        if (mn) {
          mn->dump(OS);
        } else {
          OS << "<NULL/>";
        }
      }
      OS << "</s" << (unsigned)kind << '>';
    }
  }
};
}

static std::unique_ptr<SPNode>
convertClientNode(swiftparse_client_node_t client_node) {
  return std::unique_ptr<SPNode>((SPNode*)client_node);
}

static size_t trivialLen(ArrayRef<swiftparse_trivia_piece_t> trivia) {
  size_t len = 0;
  for (const auto &piece : trivia) {
    len += piece.length;
  }
  return len;
}

static swiftparse_client_node_t
makeNode(const swiftparse_syntax_node_t *raw_node, StringRef source) {
  SPNode *node = new SPNode();
  node->kind = raw_node->kind;
  if (raw_node->kind == 0) {
    auto range = raw_node->token_data.range;
    auto nodeText = source.substr(range.offset, range.length);
    node->tokKind = raw_node->token_data.kind;
    size_t leadingTriviaLen =
      trivialLen(makeArrayRef(raw_node->token_data.leading_trivia,
                              raw_node->token_data.leading_trivia_count));
    size_t trailingTriviaLen =
      trivialLen(makeArrayRef(raw_node->token_data.trailing_trivia,
                              raw_node->token_data.trailing_trivia_count));
    node->leadingTriviaText = nodeText.take_front(leadingTriviaLen);
    node->tokenText = nodeText.substr(
        leadingTriviaLen, range.length - leadingTriviaLen - trailingTriviaLen);
    node->trailingTriviaText = nodeText.take_back(trailingTriviaLen);
  } else {
    for (unsigned i = 0, e = raw_node->layout_data.nodes_count; i != e; ++i) {
      auto subnode = convertClientNode(raw_node->layout_data.nodes[i]);
      node->members.push_back(std::move(subnode));
    }
  }
  return node;
}

static swiftparse_client_node_t
parse(StringRef source, swiftparse_node_handler_t node_handler,
      StringRef swift_version, bool enable_bare_slash_regex,
      swiftparse_diagnostic_handler_t diag_handler = nullptr) {
  swiftparse_parser_t parser = swiftparse_parser_create();
  if (!swift_version.empty()) {
    swiftparse_parser_set_language_version(parser, swift_version.str().c_str());
  }
  swiftparse_parser_set_enable_bare_slash_regex_literal(
      parser, enable_bare_slash_regex);
  swiftparse_parser_set_node_handler(parser, node_handler);
  swiftparse_parser_set_diagnostic_handler(parser, diag_handler);
  swiftparse_client_node_t top =
      swiftparse_parse_string(parser, source.data(), source.size());
  swiftparse_parser_dispose(parser);
  return top;
}

static int dumpTree(StringRef source, StringRef swiftVersion,
                    bool enableBareSlashRegex) {
  swiftparse_node_handler_t nodeHandler =
    ^swiftparse_client_node_t(const swiftparse_syntax_node_t *raw_node) {
      return makeNode(raw_node, source);
    };

  std::unique_ptr<SPNode> top = convertClientNode(
      parse(source, nodeHandler, swiftVersion, enableBareSlashRegex));
  top->dump(outs());

  return 0;
}

static void printLineColumn(SourceMgr &SM, unsigned BufferId, unsigned Offset,
                            llvm::raw_ostream &OS) {
  auto Pair = SM.getLineAndColumn(SMLoc::getFromPointer(SM.getBufferInfo(BufferId).
    Buffer->getBuffer().data() + Offset));
  OS << Pair.first << ":" << Pair.second;
}

static void printRange(SourceMgr &SM, unsigned BufferId, swiftparse_range_t Range,
                       llvm::raw_ostream &OS) {
  OS << "(";
  printLineColumn(SM, BufferId, Range.offset, OS);
  OS << ",";
  printLineColumn(SM, BufferId, Range.offset + Range.length, OS);
  OS << ")";
}

struct PrintDiagData {
  unsigned Error = 0;
  unsigned Warning = 0;
  unsigned Note = 0;
  ~PrintDiagData() {
    outs() << Error << " error(s) " << Warning << " warnings(s) "
           << Note << " note(s)\n";
  }
};

static void printDiagInfo(const swiftparser_diagnostic_t diag,
                          llvm::SourceMgr &SM, unsigned BufferId,
                          PrintDiagData &Data) {
  printLineColumn(SM, BufferId, swiftparse_diagnostic_get_source_loc(diag), outs());
  switch(swiftparse_diagnostic_get_severity(diag)) {
  case SWIFTPARSER_DIAGNOSTIC_SEVERITY_ERROR:
    Data.Error ++;
    outs() << " Error: ";
    break;
  case SWIFTPARSER_DIAGNOSTIC_SEVERITY_WARNING:
    Data.Warning ++;
    outs() << " Warning: ";
    break;
  case SWIFTPARSER_DIAGNOSTIC_SEVERITY_NOTE:
    Data.Note ++;
    outs() << " Note: ";
    break;
  }
  outs() << swiftparse_diagnostic_get_message(diag) << "\n";
  for(unsigned i = 0, n = swiftparse_diagnostic_get_range_count(diag); i < n; i ++) {
    auto range = swiftparse_diagnostic_get_range(diag, i);
    outs() << "Highlight range:";
    printRange(SM, BufferId, range, outs());
    outs() << "\n";
  }
  for(unsigned i = 0, n = swiftparse_diagnostic_get_fixit_count(diag); i < n; i ++) {
    auto fixit = swiftparse_diagnostic_get_fixit(diag, i);
    printRange(SM, BufferId, fixit.range, outs());
    outs() << " Fixit: \"" << fixit.text << "\"\n";
  }
}

static int dumpDiagnostics(StringRef source, llvm::SourceMgr &SM,
                           unsigned BufferId, StringRef swiftVersion,
                           bool enableBareSlashRegex) {
  swiftparse_node_handler_t nodeHandler =
  ^swiftparse_client_node_t(const swiftparse_syntax_node_t *raw_node) {
    return makeNode(raw_node, source);
  };
  std::shared_ptr<PrintDiagData> pData = std::make_shared<PrintDiagData>();
  convertClientNode(parse(
      source, nodeHandler, swiftVersion, enableBareSlashRegex,
      ^(const swiftparser_diagnostic_t diag) {
        printDiagInfo(diag, SM, BufferId, const_cast<PrintDiagData &>(*pData));
      }));
  return 0;
}

static void printTimeRecord(unsigned numInvoks, const TimeRecord &total,
                            raw_ostream &OS) {
  if (total.getUserTime())
    OS << "   ---User Time---";
  if (total.getSystemTime())
    OS << "   --System Time--";
  if (total.getProcessTime())
    OS << "   --User+System--";
  OS << "   ---Wall Time---";
  if (total.getMemUsed())
    OS << "  ---Mem---";
  OS << '\n';

  auto printVal = [&](double Val) {
    OS << format("  %8.5f        ", Val);
  };

  printVal(total.getUserTime()/numInvoks);
  printVal(total.getSystemTime()/numInvoks);
  printVal(total.getProcessTime()/numInvoks);
  printVal(total.getWallTime()/numInvoks);

  OS << '\n';
}

static int timeParsing(StringRef source, unsigned numInvoks,
                       StringRef swiftVersion, bool enableBareSlashRegex) {
  swiftparse_node_handler_t nodeHandler =
    ^swiftparse_client_node_t(const swiftparse_syntax_node_t *raw_node) {
      return nullptr;
    };

  Timer timer;
  timer.startTimer();
  for (unsigned i = 0; i != numInvoks; ++i) {
    parse(source, nodeHandler, swiftVersion, enableBareSlashRegex);
  }
  timer.stopTimer();

  printTimeRecord(numInvoks, timer.getTotalTime(), outs());
  return 0;
}

int main(int argc, char *argv[]) {
  PROGRAM_START(argc, argv);
  cl::ParseCommandLineOptions(argc, argv, "Swift Syntax Parser Test\n");

  StringRef fname = options::Filename[0];
  llvm::SourceMgr SM;
  auto fileBufOrErr = MemoryBuffer::getFile(fname);
  if (!fileBufOrErr) {
    errs() << "error opening file '" << fname << "': "
                 << fileBufOrErr.getError().message();
    return 1;
  }
  StringRef source = fileBufOrErr.get()->getBuffer();
  auto BufferId = SM.AddNewSourceBuffer(std::move(*fileBufOrErr), SMLoc());
  switch (options::Action) {
  case ActionType::DumpTree:
    return dumpTree(source, options::SwiftVersion,
                    options::EnableBareSlashRegex);
  case ActionType::Time:
    return timeParsing(source, options::NumParses, options::SwiftVersion,
                       options::EnableBareSlashRegex);
  case ActionType::Diagnostics:
    return dumpDiagnostics(source, SM, BufferId, options::SwiftVersion,
                           options::EnableBareSlashRegex);
  }
}