[clangd] Define a compact binary serialization fomat for symbol slab/index.

Summary:
This is intended to replace the current YAML format for general use.
It's ~10x more compact than YAML, and ~40% more compact than gzipped YAML:
  llvmidx.riff = 20M, llvmidx.yaml = 272M, llvmidx.yaml.gz = 32M
It's also simpler/faster to read and write.

The format is a RIFF container (chunks of (type, size, data)) with:
 - a compressed string table
 - simple binary encoding of symbols (with varints for compactness)
It can be extended to include occurrences, Dex posting lists, etc.

There's no rich backwards-compatibility scheme, but a version number is included
so we can detect incompatible files and do ad-hoc back-compat.

Alternatives considered:
 - compressed YAML or JSON: bulky and slow to load
 - llvm bitstream: confusing model and libraries are hard to use. My attempt
   produced slightly larger files, and the code was longer and slower.
 - protobuf or similar: would be really nice (esp for back-compat) but the
   dependency is a big hassle
 - ad-hoc binary format without a container: it seems clear we're going
   to add posting lists and occurrences here, and that they will benefit
   from sharing a string table. The container makes it easy to debug
   these pieces in isolation, and make them optional.

Reviewers: ioeric

Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, mgrang, arphaman, kadircet, cfe-commits

Differential Revision: https://reviews.llvm.org/D51585

llvm-svn: 341375

Author: sam-mccall

clang-tools-extra/clangd/CMakeLists.txt

@@ -29,6 +29,7 @@ add_clang_library(clangDaemon
   Protocol.cpp
   ProtocolHandlers.cpp
   Quality.cpp
+  RIFF.cpp
   SourceCode.cpp
   Threading.cpp
   Trace.cpp
@@ -41,6 +42,7 @@ add_clang_library(clangDaemon
   index/Index.cpp
   index/MemIndex.cpp
   index/Merge.cpp
+  index/Serialization.cpp
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 

clang-tools-extra/clangd/RIFF.cpp

@@ -0,0 +1,88 @@
+//===--- RIFF.cpp - Binary container file format --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RIFF.h"
+#include "llvm/Support/Endian.h"
+
+using namespace llvm;
+namespace clang {
+namespace clangd {
+namespace riff {
+
+static Error makeError(const char *Msg) {
+  return createStringError(inconvertibleErrorCode(), Msg);
+}
+
+Expected<Chunk> readChunk(StringRef &Stream) {
+  if (Stream.size() < 8)
+    return makeError("incomplete chunk header");
+  Chunk C;
+  std::copy(Stream.begin(), Stream.begin() + 4, C.ID.begin());
+  Stream = Stream.drop_front(4);
+  uint32_t Len = support::endian::read32le(Stream.take_front(4).begin());
+  Stream = Stream.drop_front(4);
+  if (Stream.size() < Len)
+    return makeError("truncated chunk");
+  C.Data = Stream.take_front(Len);
+  Stream = Stream.drop_front(Len);
+  if (Len % 2 & !Stream.empty()) { // Skip padding byte.
+    if (Stream.front())
+      return makeError("nonzero padding byte");
+    Stream = Stream.drop_front();
+  }
+  return C;
+};
+
+raw_ostream &operator<<(raw_ostream &OS, const Chunk &C) {
+  OS.write(C.ID.begin(), C.ID.size());
+  char Size[4];
+  llvm::support::endian::write32le(Size, C.Data.size());
+  OS.write(Size, sizeof(Size));
+  OS << C.Data;
+  if (C.Data.size() % 2)
+    OS.write(0);
+  return OS;
+}
+
+llvm::Expected<File> readFile(llvm::StringRef Stream) {
+  auto RIFF = readChunk(Stream);
+  if (!RIFF)
+    return RIFF.takeError();
+  if (RIFF->ID != fourCC("RIFF"))
+    return makeError("not a RIFF container");
+  if (RIFF->Data.size() < 4)
+    return makeError("RIFF chunk too short");
+  File F;
+  std::copy(RIFF->Data.begin(), RIFF->Data.begin() + 4, F.Type.begin());
+  for (llvm::StringRef Body = RIFF->Data.drop_front(4); !Body.empty();)
+    if (auto Chunk = readChunk(Body)) {
+      F.Chunks.push_back(*Chunk);
+    } else
+      return Chunk.takeError();
+  return F;
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const File &F) {
+  // To avoid copies, we serialize the outer RIFF chunk "by hand".
+  size_t DataLen = 4; // Predict length of RIFF chunk data.
+  for (const auto &C : F.Chunks)
+    DataLen += 4 + 4 + C.Data.size() + (C.Data.size() % 2);
+  OS << "RIFF";
+  char Size[4];
+  llvm::support::endian::write32le(Size, DataLen);
+  OS.write(Size, sizeof(Size));
+  OS.write(F.Type.begin(), F.Type.size());
+  for (const auto &C : F.Chunks)
+    OS << C;
+  return OS;
+}
+
+} // namespace riff
+} // namespace clangd
+} // namespace clang

clang-tools-extra/clangd/RIFF.h

@@ -0,0 +1,81 @@
+//===--- RIFF.h - Binary container file format -------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Tools for reading and writing data in RIFF containers.
+//
+// A chunk consists of:
+//   - ID      : char[4]
+//   - Length  : uint32
+//   - Data    : byte[Length]
+//   - Padding : byte[Length % 2]
+// The semantics of a chunk's Data are determined by its ID.
+// The format makes it easy to skip over uninteresting or unknown chunks.
+//
+// A RIFF file is a single chunk with ID "RIFF". Its Data is:
+//   - Type    : char[4]
+//   - Chunks  : chunk[]
+//
+// This means that a RIFF file consists of:
+//   - "RIFF"          : char[4]
+//   - File length - 8 : uint32
+//   - File type       : char[4]
+//   - Chunks          : chunk[]
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_RIFF_H
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ScopedPrinter.h"
+#include <array>
+
+namespace clang {
+namespace clangd {
+namespace riff {
+
+// A FourCC identifies a chunk in a file, or the type of file itself.
+using FourCC = std::array<char, 4>;
+// Get a FourCC from a string literal, e.g. fourCC("RIFF").
+inline constexpr FourCC fourCC(const char (&Literal)[5]) {
+  return FourCC{{Literal[0], Literal[1], Literal[2], Literal[3]}};
+}
+// A chunk is a section in a RIFF container.
+struct Chunk {
+  FourCC ID;
+  llvm::StringRef Data;
+};
+inline bool operator==(const Chunk &L, const Chunk &R) {
+  return std::tie(L.ID, L.Data) == std::tie(R.ID, R.Data);
+}
+// A File is a RIFF container, which is a typed chunk sequence.
+struct File {
+  FourCC Type;
+  std::vector<Chunk> Chunks;
+};
+inline bool operator==(const File &L, const File &R) {
+  return std::tie(L.Type, L.Chunks) == std::tie(R.Type, R.Chunks);
+}
+
+// Reads a single chunk from the start of Stream.
+// Stream is updated to exclude the consumed chunk.
+llvm::Expected<Chunk> readChunk(llvm::StringRef &Stream);
+
+// Serialize a single chunk to OS.
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Chunk &);
+
+// Parses a RIFF file consisting of a single RIFF chunk.
+llvm::Expected<File> readFile(llvm::StringRef Stream);
+
+// Serialize a RIFF file (i.e. a single RIFF chunk) to OS.
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const File &);
+
+} // namespace riff
+} // namespace clangd
+} // namespace clang
+#endif

clang-tools-extra/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp

@@ -7,15 +7,16 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// GlobalSymbolBuilder is a tool to generate YAML-format symbols across the
-// whole project. This tools is for **experimental** only. Don't use it in
-// production code.
+// GlobalSymbolBuilder is a tool to extract symbols from a whole project.
+// This tool is **experimental** only. Don't use it in production code.
 //
 //===----------------------------------------------------------------------===//
 
+#include "RIFF.h"
 #include "index/CanonicalIncludes.h"
 #include "index/Index.h"
 #include "index/Merge.h"
+#include "index/Serialization.h"
 #include "index/SymbolCollector.h"
 #include "index/SymbolYAML.h"
 #include "clang/Frontend/CompilerInstance.h"
@@ -59,6 +60,14 @@ static llvm::cl::opt<bool> MergeOnTheFly(
         "MapReduce."),
     llvm::cl::init(true), llvm::cl::Hidden);
 
+enum class Format { YAML, Binary };
+static llvm::cl::opt<Format>
+    Format("format", llvm::cl::desc("Format of the index to be written"),
+           llvm::cl::values(
+               clEnumValN(Format::YAML, "yaml", "human-readable YAML format"),
+               clEnumValN(Format::Binary, "binary", "binary RIFF format")),
+           llvm::cl::init(Format::YAML));
+
 /// Responsible for aggregating symbols from each processed file and producing
 /// the final results. All methods in this class must be thread-safe,
 /// 'consumeSymbols' may be called from multiple threads.
@@ -210,8 +219,8 @@ int main(int argc, const char **argv) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   const char *Overview = R"(
-  This is an **experimental** tool to generate YAML-format project-wide symbols
-  for clangd (global code completion). It would be changed and deprecated
+  This is an **experimental** tool to extract symbols from a whole project
+  for clangd (global code completion). It will be changed and deprecated
   eventually. Don't use it in production code!
 
   Example usage for building index for the whole project using CMake compile
@@ -262,7 +271,16 @@ int main(int argc, const char **argv) {
   }
   // Reduce phase: combine symbols with the same IDs.
   auto UniqueSymbols = Consumer->mergeResults();
-  // Output phase: emit YAML for result symbols.
-  SymbolsToYAML(UniqueSymbols, llvm::outs());
+  // Output phase: emit result symbols.
+  switch (clang::clangd::Format) {
+  case clang::clangd::Format::YAML:
+    SymbolsToYAML(UniqueSymbols, llvm::outs());
+    break;
+  case clang::clangd::Format::Binary: {
+    clang::clangd::IndexFileOut Out;
+    Out.Symbols = &UniqueSymbols;
+    llvm::outs() << Out;
+  }
+  }
   return 0;
 }

clang-tools-extra/clangd/index/Index.cpp

@@ -10,6 +10,7 @@
 #include "Index.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/SHA1.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -28,21 +29,20 @@ SymbolID::SymbolID(StringRef USR)
     : HashValue(SHA1::hash(arrayRefFromStringRef(USR))) {}
 
 raw_ostream &operator<<(raw_ostream &OS, const SymbolID &ID) {
-  OS << toHex(toStringRef(ID.HashValue));
-  return OS;
+  return OS << toHex(ID.raw());
 }
 
-std::string SymbolID::str() const {
-  std::string ID;
-  llvm::raw_string_ostream OS(ID);
-  OS << *this;
-  return OS.str();
+SymbolID SymbolID::fromRaw(llvm::StringRef Raw) {
+  SymbolID ID;
+  assert(Raw.size() == RawSize);
+  memcpy(ID.HashValue.data(), Raw.data(), RawSize);
+  return ID;
 }
 
+std::string SymbolID::str() const { return toHex(raw()); }
+
 void operator>>(StringRef Str, SymbolID &ID) {
-  std::string HexString = fromHex(Str);
-  assert(HexString.size() == ID.HashValue.size());
-  std::copy(HexString.begin(), HexString.end(), ID.HashValue.begin());
+  ID = SymbolID::fromRaw(fromHex(Str));
 }
 
 raw_ostream &operator<<(raw_ostream &OS, SymbolOrigin O) {
@@ -78,34 +78,18 @@ SymbolSlab::const_iterator SymbolSlab::find(const SymbolID &ID) const {
 }
 
 // Copy the underlying data of the symbol into the owned arena.
-static void own(Symbol &S, llvm::UniqueStringSaver &Strings,
-                BumpPtrAllocator &Arena) {
-  // Intern replaces V with a reference to the same string owned by the arena.
-  auto Intern = [&](StringRef &V) { V = Strings.save(V); };
-
-  // We need to copy every StringRef field onto the arena.
-  Intern(S.Name);
-  Intern(S.Scope);
-  Intern(S.CanonicalDeclaration.FileURI);
-  Intern(S.Definition.FileURI);
-
-  Intern(S.Signature);
-  Intern(S.CompletionSnippetSuffix);
-
-  Intern(S.Documentation);
-  Intern(S.ReturnType);
-  for (auto &I : S.IncludeHeaders)
-    Intern(I.IncludeHeader);
+static void own(Symbol &S, llvm::UniqueStringSaver &Strings) {
+  visitStrings(S, [&](StringRef &V) { V = Strings.save(V); });
 }
 
 void SymbolSlab::Builder::insert(const Symbol &S) {
   auto R = SymbolIndex.try_emplace(S.ID, Symbols.size());
   if (R.second) {
     Symbols.push_back(S);
-    own(Symbols.back(), UniqueStrings, Arena);
+    own(Symbols.back(), UniqueStrings);
   } else {
     auto &Copy = Symbols[R.first->second] = S;
-    own(Copy, UniqueStrings, Arena);
+    own(Copy, UniqueStrings);
   }
 }
 
@@ -118,7 +102,7 @@ SymbolSlab SymbolSlab::Builder::build() && {
   BumpPtrAllocator NewArena;
   llvm::UniqueStringSaver Strings(NewArena);
   for (auto &S : Symbols)
-    own(S, Strings, NewArena);
+    own(S, Strings);
   return SymbolSlab(std::move(NewArena), std::move(Symbols));
 }