Symbol graph support

Adds a tool `swift-symbolgraph-extract` that reads an existing Swift
module and prints a platform- and language-agnostic JSON description of
the module, primarly for documentation.

Adds a small sub-library `SymbolGraphGen` which houses the core
implementation for collecting relevant information about declarations.
The main entry point is integrated directly into the driver as a mode:
the tool is meant to be run outside of the normal edit-compile-run/test
workflow to avoid impacting build times.

Along with common options for other tools, unique options include
`pretty-print` for debugging, and a `minimum-access-level` options for
including internal documentation.

A symbol graph is a directed graph where the nodes are symbols in a
module and the edges are relationships between them. For example, a
`struct S` may have a member `var x`. The graph would have two nodes for
`S` and `x`, and one "member-of" relationship edge. Other relationship
kinds include "inherits-from" or "conforms to". The data format for a
symbol graph is still under development and may change without notice
until a specificiation and versioning scheme is published.

Various aspects about a symbol are recorded in the nodes, such as
availability, documentation comments, or data needed for printing the
shapes of declarations without having to understand specifics about the
langauge.

Implicit and public-underscored stdlib declarations are not included by
default.

rdar://problem/55346798

Author: bitjammer

include/swift/AST/Decl.h

@@ -875,6 +875,8 @@ class alignas(1 << DeclAlignInBits) Decl {
   LLVM_READONLY
   const GenericContext *getAsGenericContext() const;
 
+  bool hasUnderscoredNaming() const;
+
   bool isPrivateStdlibDecl(bool treatNonBuiltinProtocolsAsPublic = true) const;
 
   AvailabilityContext getAvailabilityForLinkage() const;

include/swift/AST/PrintOptions.h

@@ -380,6 +380,13 @@ struct PrintOptions {
   ArgAndParamPrintingMode ArgAndParamPrinting =
       ArgAndParamPrintingMode::MatchSource;
 
+  /// Whether to print the default argument value string
+  /// representation.
+  bool PrintDefaultArgumentValue = true;
+
+  /// Whether to print "_" placeholders for empty arguments.
+  bool PrintEmptyArgumentNames = true;
+
   /// Whether to print documentation comments attached to declarations.
   /// Note that this may print documentation comments from related declarations
   /// (e.g. the overridden method in the superclass) if such comment is found.

include/swift/Driver/Driver.h

@@ -158,7 +158,8 @@ class Driver {
     Interactive,     // swift
     Batch,           // swiftc
     AutolinkExtract, // swift-autolink-extract
-    SwiftIndent      // swift-indent
+    SwiftIndent,     // swift-indent
+    SymbolGraph      // swift-symbolgraph
   };
 
   class InputInfoMap;

include/swift/SymbolGraphGen/SymbolGraphGen.h

@@ -0,0 +1,41 @@
+//===--- swift_indent_main.cpp - Swift code formatting tool ---------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Triple.h"
+#include "swift/AST/AttrKind.h"
+
+namespace swift {
+
+class ModuleDecl;
+
+namespace symbolgraphgen {
+
+struct SymbolGraphOptions {
+  /// The path to output the symbol graph JSON.
+  StringRef OutputPath;
+
+  /// The target of the module.
+  llvm::Triple Target;
+
+  /// Pretty-print the JSON with newlines and indentation.
+  bool PrettyPrint;
+
+  /// The minimum access level that symbols must have in order to be
+  /// included in the graph.
+  AccessLevel MinimumAccessLevel;
+};
+
+/// Emit a Symbol Graph JSON file for a module.
+int emitSymbolGraphForModule(ModuleDecl *M, const SymbolGraphOptions &Options);
+
+} // end namespace symbolgraphgen
+} // end namespace swift

lib/AST/ASTPrinter.cpp

@@ -2632,6 +2632,9 @@ void PrintAST::printOneParameter(const ParamDecl *param,
       // Else, print the argument only.
       LLVM_FALLTHROUGH;
     case PrintOptions::ArgAndParamPrintingMode::ArgumentOnly:
+      if (ArgName.empty() && !Options.PrintEmptyArgumentNames) {
+        return;
+      }
       Printer.printName(ArgName, PrintNameContext::FunctionParameterExternal);
 
       if (!ArgNameIsAPIByDefault && !ArgName.empty())
@@ -2686,7 +2689,7 @@ void PrintAST::printOneParameter(const ParamDecl *param,
   if (param->isVariadic())
     Printer << "...";
 
-  if (param->isDefaultArgument()) {
+  if (param->isDefaultArgument() && Options.PrintDefaultArgumentValue) {
     SmallString<128> scratch;
     auto defaultArgStr = param->getDefaultValueStringRepresentation(scratch);
 

lib/AST/Decl.cpp

@@ -697,6 +697,56 @@ bool ParameterList::hasInternalParameter(StringRef Prefix) const {
   return false;
 }
 
+bool Decl::hasUnderscoredNaming() const {
+  const Decl *D = this;
+  if (const auto AFD = dyn_cast<AbstractFunctionDecl>(D)) {
+    // If it's a function with a parameter with leading underscore, it's a
+    // private function.
+    if (AFD->getParameters()->hasInternalParameter("_")) {
+      return true;
+    }
+  }
+
+  if (const auto SubscriptD = dyn_cast<SubscriptDecl>(D)) {
+    if (SubscriptD->getIndices()->hasInternalParameter("_")) {
+      return true;
+    }
+  }
+
+  if (const auto PD = dyn_cast<ProtocolDecl>(D)) {
+    if (PD->getAttrs().hasAttribute<ShowInInterfaceAttr>()) {
+      return false;
+    }
+    StringRef NameStr = PD->getNameStr();
+    if (NameStr.startswith("_Builtin")) {
+      return true;
+    }
+    if (NameStr.startswith("_ExpressibleBy")) {
+      return true;
+    }
+  }
+
+  if (const auto ImportD = dyn_cast<ImportDecl>(D)) {
+    if (const auto *Mod = ImportD->getModule()) {
+      if (Mod->isSwiftShimsModule()) {
+        return true;
+      }
+    }
+  }
+
+  const auto VD = dyn_cast<ValueDecl>(D);
+  if (!VD || !VD->hasName()) {
+    return false;
+  }
+
+  if (!VD->getBaseName().isSpecial() &&
+      VD->getBaseName().getIdentifier().str().startswith("_")) {
+    return true;
+  }
+
+  return false;
+}
+
 bool Decl::isPrivateStdlibDecl(bool treatNonBuiltinProtocolsAsPublic) const {
   const Decl *D = this;
   if (auto ExtD = dyn_cast<ExtensionDecl>(D)) {
@@ -718,47 +768,12 @@ bool Decl::isPrivateStdlibDecl(bool treatNonBuiltinProtocolsAsPublic) const {
       FU->getKind() != FileUnitKind::SerializedAST)
     return false;
 
-  if (auto AFD = dyn_cast<AbstractFunctionDecl>(D)) {
-    // If it's a function with a parameter with leading underscore, it's a
-    // private function.
-    if (AFD->getParameters()->hasInternalParameter("_"))
-      return true;
-  }
-
-  if (auto SubscriptD = dyn_cast<SubscriptDecl>(D)) {
-    if (SubscriptD->getIndices()->hasInternalParameter("_"))
-      return true;
-  }
-
   if (auto PD = dyn_cast<ProtocolDecl>(D)) {
-    if (PD->getAttrs().hasAttribute<ShowInInterfaceAttr>())
-      return false;
-    StringRef NameStr = PD->getNameStr();
-    if (NameStr.startswith("_Builtin"))
-      return true;
-    if (NameStr.startswith("_ExpressibleBy"))
-      return true;
     if (treatNonBuiltinProtocolsAsPublic)
       return false;
   }
 
-  if (auto ImportD = dyn_cast<ImportDecl>(D)) {
-    if (auto *Mod = ImportD->getModule()) {
-      if (Mod->isSwiftShimsModule())
-        return true;
-    }
-  }
-
-  auto VD = dyn_cast<ValueDecl>(D);
-  if (!VD || !VD->hasName())
-    return false;
-
-  // If the name has leading underscore then it's a private symbol.
-  if (!VD->getBaseName().isSpecial() &&
-      VD->getBaseName().getIdentifier().str().startswith("_"))
-    return true;
-
-  return false;
+  return hasUnderscoredNaming();
 }
 
 AvailabilityContext Decl::getAvailabilityForLinkage() const {

lib/CMakeLists.txt

@@ -42,6 +42,7 @@ add_subdirectory(SwiftRemoteMirror)
 add_subdirectory(SIL)
 add_subdirectory(SILGen)
 add_subdirectory(SILOptimizer)
+add_subdirectory(SymbolGraphGen)
 add_subdirectory(Syntax)
 add_subdirectory(SyntaxParse)
 add_subdirectory(TBDGen)

lib/Driver/Driver.cpp

@@ -98,6 +98,7 @@ void Driver::parseDriverKind(ArrayRef<const char *> Args) {
   .Case("swiftc", DriverKind::Batch)
   .Case("swift-autolink-extract", DriverKind::AutolinkExtract)
   .Case("swift-indent", DriverKind::SwiftIndent)
+  .Case("swift-symbolgraph-extract", DriverKind::SymbolGraph)
   .Default(None);
 
   if (Kind.hasValue())
@@ -3252,6 +3253,7 @@ void Driver::printHelp(bool ShowHidden) const {
   case DriverKind::Batch:
   case DriverKind::AutolinkExtract:
   case DriverKind::SwiftIndent:
+  case DriverKind::SymbolGraph:
     ExcludedFlagsBitmask |= options::NoBatchOption;
     break;
   }

lib/Markup/LineList.cpp

@@ -115,8 +115,6 @@ LineList MarkupContext::getLineList(swift::RawComment RC) {
       // Determine if we have leading decorations in this block comment.
       bool HasASCIIArt = false;
       if (swift::startsWithNewline(Cleaned)) {
-        Builder.addLine(Cleaned.substr(0, 0), { C.Range.getStart(),
-                                                C.Range.getStart() });
         unsigned NewlineBytes = swift::measureNewline(Cleaned);
         Cleaned = Cleaned.drop_front(NewlineBytes);
         CleanedStartLoc = CleanedStartLoc.getAdvancedLocOrInvalid(NewlineBytes);

lib/SymbolGraphGen/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_swift_host_library(swiftSymbolGraphGen STATIC
+  DeclarationFragmentPrinter.cpp
+  Edge.cpp
+  JSON.cpp
+  Symbol.cpp
+  SymbolGraph.cpp
+  SymbolGraphGen.cpp
+  SymbolGraphASTWalker.cpp)
+
+target_link_libraries(swiftSymbolGraphGen
+  swiftAST
+  swiftFrontend
+  swiftMarkup)

lib/SymbolGraphGen/DeclarationFragmentPrinter.cpp

@@ -0,0 +1,140 @@
+//===--- DeclarationFragmentPrinter.cpp - Declaration Fragment Printer ----===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#include "DeclarationFragmentPrinter.h"
+#include "SymbolGraphASTWalker.h"
+
+using namespace swift;
+using namespace symbolgraphgen;
+
+void DeclarationFragmentPrinter::openFragment(FragmentKind Kind) {
+  assert(Kind != FragmentKind::None);
+  if (this->Kind != Kind) {
+    closeFragment();
+    this->Kind = Kind,
+    Spelling.clear();
+    USR.clear();
+  }
+}
+
+StringRef
+DeclarationFragmentPrinter::getKindSpelling(FragmentKind Kind) const {
+  switch (Kind) {
+    case FragmentKind::Keyword:
+      return "keyword";
+    case FragmentKind::Attribute:
+      return "attribute";
+    case FragmentKind::NumberLiteral:
+      return "number";
+    case FragmentKind::StringLiteral:
+      return "string";
+    case FragmentKind::Identifier:
+      return "identifier";
+    case FragmentKind::TypeIdentifier:
+      return "typeIdentifier";
+    case FragmentKind::GenericParameter:
+      return "genericParameter";
+    case FragmentKind::Text:
+      return "text";
+    case FragmentKind::None:
+      llvm_unreachable("Fragment kind of 'None' has no spelling");
+  }
+}
+
+void DeclarationFragmentPrinter::closeFragment() {
+  if (Kind == FragmentKind::None) {
+    return;
+  }
+
+  if (!Spelling.empty()) {
+    OS.object([&](){
+      OS.attribute("kind", getKindSpelling(Kind));
+      OS.attribute("spelling", Spelling.str());
+      if (!USR.empty()) {
+        OS.attribute("preciseIdentifier", USR.str());
+      }
+    });
+  }
+
+  Spelling.clear();
+  USR.clear();
+  Kind = FragmentKind::None;
+}
+
+void DeclarationFragmentPrinter::printDeclLoc(const Decl *D) {
+  switch (D->getKind()) {
+    case DeclKind::Constructor:
+    case DeclKind::Destructor:
+    case DeclKind::Subscript:
+      openFragment(FragmentKind::Keyword);
+      break;
+    default:
+      openFragment(FragmentKind::Identifier);
+      break;
+  }
+}
+
+void
+DeclarationFragmentPrinter::printNamePre(PrintNameContext Context) {
+  switch (Context) {
+  case PrintNameContext::Keyword:
+    openFragment(FragmentKind::Keyword);
+    break;
+  case PrintNameContext::GenericParameter:
+    openFragment(FragmentKind::GenericParameter);
+    break;
+  case PrintNameContext::Attribute:
+    openFragment(FragmentKind::Attribute);
+    break;
+  case PrintNameContext::ClassDynamicSelf:
+  case PrintNameContext::FunctionParameterExternal:
+    openFragment(FragmentKind::Identifier);
+    break;
+  case PrintNameContext::FunctionParameterLocal:
+    openFragment(FragmentKind::Identifier);
+    break;
+  case PrintNameContext::TupleElement:
+  case PrintNameContext::TypeMember:
+  case PrintNameContext::Normal:
+    break;
+  }
+}
+
+void DeclarationFragmentPrinter::printStructurePre(PrintStructureKind Kind,
+                                                   const Decl *D) {
+  switch (Kind) {
+    case PrintStructureKind::NumberLiteral:
+      openFragment(FragmentKind::NumberLiteral);
+      break;
+    case PrintStructureKind::StringLiteral:
+      openFragment(FragmentKind::StringLiteral);
+      break;
+    default:
+      break;
+  }
+}
+
+void DeclarationFragmentPrinter::printTypeRef(Type T, const TypeDecl *RefTo,
+    Identifier Name,
+    PrintNameContext NameContext) {
+  openFragment(FragmentKind::TypeIdentifier);
+  printText(Name.str());
+  USR = Walker.getUSR(RefTo);
+  closeFragment();
+}
+
+void DeclarationFragmentPrinter::printText(StringRef Text) {
+  if (Kind == FragmentKind::None) {
+    openFragment(FragmentKind::Text);
+  }
+  Spelling.append(Text);
+}