[CursorInfo] Add Clang documentation to SymbolGraph output

This currently doesn't check for inherited docs, ie. either the
imported declaration has docs or it doesn't. There's also a few odd
cases with mixed doc types and when each line is prefixed with '*', but
it's good enough for an initial implementation.

Moves UTF8 sanitisation out of ASTPrinter.h and into Unicode.h so that
it can be used here as well.

Resolves rdar://91388603.

Author: bnbarham

include/swift/AST/ASTPrinter.h

@@ -327,9 +327,6 @@ class ASTPrinter {
     printStructurePre(Kind, D);
   }
 
-  /// To sanitize a malformed utf8 string to a well-formed one.
-  static std::string sanitizeUtf8(StringRef Text);
-
 private:
   virtual void anchor();
 };

include/swift/Basic/Unicode.h

@@ -74,6 +74,9 @@ unsigned extractFirstUnicodeScalar(StringRef S);
 /// unit (Unicode scalar).
 bool isWellFormedUTF8(StringRef S);
 
+/// Replaces any ill-formed subsequences with `u8"\ufffd"`.
+std::string sanitizeUTF8(StringRef Text);
+
 } // end namespace unicode
 } // end namespace swift
 

include/swift/SymbolGraphGen/SymbolGraphOptions.h

@@ -45,6 +45,9 @@ struct SymbolGraphOptions {
 
   /// Whether to emit symbols with SPI information.
   bool IncludeSPISymbols;
+
+  /// Whether to include documentation for clang nodes or not.
+  bool IncludeClangDocs;
 };
 
 } // end namespace symbolgraphgen

lib/AST/ASTPrinter.cpp

@@ -46,6 +46,7 @@
 #include "swift/Basic/QuotedString.h"
 #include "swift/Basic/STLExtras.h"
 #include "swift/Basic/StringExtras.h"
+#include "swift/Basic/Unicode.h"
 #include "swift/ClangImporter/ClangImporterRequests.h"
 #include "swift/Config.h"
 #include "swift/Parse/Lexer.h"
@@ -302,31 +303,6 @@ bool TypeTransformContext::isPrintingSynthesizedExtension() const {
   return !Decl.isNull();
 }
 
-std::string ASTPrinter::sanitizeUtf8(StringRef Text) {
-  llvm::SmallString<256> Builder;
-  Builder.reserve(Text.size());
-  const llvm::UTF8* Data = reinterpret_cast<const llvm::UTF8*>(Text.begin());
-  const llvm::UTF8* End = reinterpret_cast<const llvm::UTF8*>(Text.end());
-  StringRef Replacement = u8"\ufffd";
-  while (Data < End) {
-    auto Step = llvm::getNumBytesForUTF8(*Data);
-    if (Data + Step > End) {
-      Builder.append(Replacement);
-      break;
-    }
-
-    if (llvm::isLegalUTF8Sequence(Data, Data + Step)) {
-      Builder.append(Data, Data + Step);
-    } else {
-
-      // If malformed, add replacement characters.
-      Builder.append(Replacement);
-    }
-    Data += Step;
-  }
-  return std::string(Builder.str());
-}
-
 void ASTPrinter::anchor() {}
 
 void ASTPrinter::printIndent() {
@@ -633,9 +609,9 @@ class PrintAST : public ASTVisitor<PrintAST> {
     bool FirstLine = true;
     for (auto Line : Lines) {
       if (FirstLine)
-        Printer << sanitizeClangDocCommentStyle(ASTPrinter::sanitizeUtf8(Line));
+        Printer << sanitizeClangDocCommentStyle(unicode::sanitizeUTF8(Line));
       else
-        Printer << ASTPrinter::sanitizeUtf8(Line);
+        Printer << unicode::sanitizeUTF8(Line);
       Printer.printNewline();
       FirstLine = false;
     }

lib/Basic/Unicode.cpp

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/Basic/Unicode.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ConvertUTF.h"
 
@@ -128,3 +129,28 @@ bool swift::unicode::isWellFormedUTF8(StringRef S) {
   const llvm::UTF8 *begin = S.bytes_begin();
   return llvm::isLegalUTF8String(&begin, S.bytes_end());
 }
+
+std::string swift::unicode::sanitizeUTF8(StringRef Text) {
+  llvm::SmallString<256> Builder;
+  Builder.reserve(Text.size());
+  const llvm::UTF8* Data = reinterpret_cast<const llvm::UTF8*>(Text.begin());
+  const llvm::UTF8* End = reinterpret_cast<const llvm::UTF8*>(Text.end());
+  StringRef Replacement = u8"\ufffd";
+  while (Data < End) {
+    auto Step = llvm::getNumBytesForUTF8(*Data);
+    if (Data + Step > End) {
+      Builder.append(Replacement);
+      break;
+    }
+
+    if (llvm::isLegalUTF8Sequence(Data, Data + Step)) {
+      Builder.append(Data, Data + Step);
+    } else {
+
+      // If malformed, add replacement characters.
+      Builder.append(Replacement);
+    }
+    Data += Step;
+  }
+  return std::string(Builder.str());
+}

lib/IDE/ModuleInterfacePrinting.cpp

@@ -22,6 +22,7 @@
 #include "swift/AST/PrintOptions.h"
 #include "swift/AST/SourceFile.h"
 #include "swift/Basic/PrimitiveParsing.h"
+#include "swift/Basic/Unicode.h"
 #include "swift/ClangImporter/ClangImporter.h"
 #include "swift/ClangImporter/ClangModule.h"
 #include "swift/Parse/Token.h"
@@ -977,7 +978,7 @@ void ClangCommentPrinter::printDeclPost(const Decl *D,
     return;
 
   for (auto CommentText : PendingComments) {
-    *this << " " << ASTPrinter::sanitizeUtf8(CommentText);
+    *this << " " << unicode::sanitizeUTF8(CommentText);
   }
   PendingComments.clear();
   if (auto ClangN = swift::ide::getEffectiveClangNode(D))
@@ -1068,7 +1069,7 @@ void ClangCommentPrinter::printComment(StringRef RawText, unsigned StartCol) {
   trimLeadingWhitespaceFromLines(RawText, WhitespaceToTrim, Lines);
 
   for (auto Line : Lines) {
-    *this << ASTPrinter::sanitizeUtf8(Line) << "\n";
+    *this << unicode::sanitizeUTF8(Line) << "\n";
     printIndent();
   }
 }

lib/SymbolGraphGen/Symbol.cpp

@@ -16,7 +16,11 @@
 #include "swift/AST/ParameterList.h"
 #include "swift/AST/RawComment.h"
 #include "swift/AST/USRGeneration.h"
+#include "swift/Basic/PrimitiveParsing.h"
 #include "swift/Basic/SourceManager.h"
+#include "swift/Basic/Unicode.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
 #include "AvailabilityMixin.h"
 #include "JSON.h"
 #include "Symbol.h"
@@ -193,6 +197,41 @@ const ValueDecl *Symbol::getDeclInheritingDocs() const {
 }
 
 void Symbol::serializeDocComment(llvm::json::OStream &OS) const {
+  if (ClangNode ClangN = VD->getClangNode()) {
+    if (!Graph->Walker.Options.IncludeClangDocs)
+      return;
+
+    if (auto *ClangD = ClangN.getAsDecl()) {
+      const clang::ASTContext &ClangContext = ClangD->getASTContext();
+      const clang::RawComment *RC =
+          ClangContext.getRawCommentForAnyRedecl(ClangD);
+      if (!RC || !RC->isDocumentation())
+        return;
+
+      // TODO: Replace this with `getFormattedLines` when it's in and add the
+      // line and column ranges. Also consider handling cross-language
+      // hierarchies, ie. if there's no comment on the ObjC decl we should
+      // look up the hierarchy (and vice versa).
+      std::string Text = RC->getFormattedText(ClangContext.getSourceManager(),
+                                              ClangContext.getDiagnostics());
+      Text = unicode::sanitizeUTF8(Text);
+
+      SmallVector<StringRef, 8> Lines;
+      splitIntoLines(Text, Lines);
+
+      OS.attributeObject("docComment", [&]() {
+        OS.attributeArray("lines", [&]() {
+          for (StringRef Line : Lines) {
+            OS.object([&](){
+              OS.attribute("text", Line);
+            });
+          }
+        });
+      });
+    }
+    return;
+  }
+
   const auto *DocCommentProvidingDecl = VD;
   if (!Graph->Walker.Options.SkipInheritedDocs) {
     DocCommentProvidingDecl = dyn_cast_or_null<ValueDecl>(