Skip to content
This repository was archived by the owner on Nov 1, 2021. It is now read-only.

Commit 407c31d

Browse files
committed
Wrap clang module files in a Mach-O, ELF, or COFF container.
This is a necessary prerequisite for debugging with modules. The .pcm files become containers that hold the serialized AST which allows us to store debug information in the module file that can be shared by all object files that were built importing the module. This reapplies r230044 with a fixed configure+make build and updated dependencies and testcase requirements. Over the last iteration this version adds - missing target requirements for testcases that specify an x86 triple, - a missing clangCodeGen.a dependency to libClang.a in the make build. rdar://problem/19104245 git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@230423 91177308-0d34-0410-b5e6-96231b3b80d8
1 parent 872f3b0 commit 407c31d

File tree

83 files changed

+456
-91
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+456
-91
lines changed

docs/PCHInternals.rst

+3
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ be included at the beginning of the translation unit. The extensions to the
6565
AST file format required for modules are discussed in the section on
6666
:ref:`modules <pchinternals-modules>`.
6767

68+
Clang's AST files are Mach-O, ELF, or COFF containers that contain a
69+
``__clangast`` section which holds the AST bitstream.
70+
6871
Clang's AST files are designed with a compact on-disk representation, which
6972
minimizes both creation time and the time required to initially load the AST
7073
file. The AST file itself contains a serialized representation of Clang's
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
//===--- CodeGen/ModuleContainerGenerator.h - Emit .pcm files ---*- C++ -*-===//
2+
//
3+
// The LLVM Compiler Infrastructure
4+
//
5+
// This file is distributed under the University of Illinois Open Source
6+
// License. See LICENSE.TXT for details.
7+
//
8+
//===----------------------------------------------------------------------===//
9+
10+
#ifndef LLVM_CLANG_CODEGEN_MODULE_CONTAINER_H
11+
#define LLVM_CLANG_CODEGEN_MODULE_CONTAINER_H
12+
13+
#include "ModuleBuilder.h"
14+
#include <string>
15+
16+
namespace llvm {
17+
class raw_ostream;
18+
}
19+
20+
namespace clang {
21+
22+
class PCHGenerator;
23+
class TargetOptions;
24+
25+
/// \brief Create a CodeGenerator instance.
26+
/// It is the responsibility of the caller to call delete on
27+
/// the allocated CodeGenerator instance.
28+
CodeGenerator *CreateModuleContainerGenerator(
29+
DiagnosticsEngine &Diags, const std::string &ModuleName,
30+
const CodeGenOptions &CGO, const TargetOptions &TO, const LangOptions &LO,
31+
llvm::raw_ostream *OS, PCHGenerator *PCHGen);
32+
}
33+
34+
#endif

include/clang/Frontend/FrontendActions.h

+8
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,14 @@ class DeclContextPrintAction : public ASTFrontendAction {
6969
StringRef InFile) override;
7070
};
7171

72+
/// \brief Emits the output of a GeneratePCHAction or GenerateModuleAction into
73+
/// a Mach-O/ELF/COFF container.
74+
class GeneratePCMContainerAction : public FrontendAction {
75+
protected:
76+
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
77+
StringRef InFile) override;
78+
};
79+
7280
class GeneratePCHAction : public ASTFrontendAction {
7381
protected:
7482
std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,

include/clang/Serialization/ASTReader.h

+4
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,10 @@ class ASTReader
11271127
public:
11281128
void ResolveImportedPath(ModuleFile &M, std::string &Filename);
11291129
static void ResolveImportedPath(std::string &Filename, StringRef Prefix);
1130+
/// \brief Initialize a BitstreamReader with the `__clangast` section from an
1131+
/// object file container found in Buffer.
1132+
static void InitStreamFileWithModule(llvm::MemoryBufferRef Buffer,
1133+
llvm::BitstreamReader &StreamFile);
11301134

11311135
private:
11321136
struct ImportedModule {

include/clang/Serialization/ASTWriter.h

+13-5
Original file line numberDiff line numberDiff line change
@@ -823,10 +823,13 @@ class PCHGenerator : public SemaConsumer {
823823
std::string OutputFile;
824824
clang::Module *Module;
825825
std::string isysroot;
826-
raw_ostream *Out;
827826
Sema *SemaPtr;
828-
SmallVector<char, 128> Buffer;
827+
// This buffer is always large, but BitstreamWriter really wants a
828+
// SmallVectorImpl<char>.
829+
SmallVector<char, 0> Buffer;
829830
llvm::BitstreamWriter Stream;
831+
std::function<void(SmallVectorImpl<char>*)>
832+
SerializationFinishedCallback;
830833
ASTWriter Writer;
831834
bool AllowASTWithErrors;
832835
bool HasEmittedPCH;
@@ -836,16 +839,21 @@ class PCHGenerator : public SemaConsumer {
836839
const ASTWriter &getWriter() const { return Writer; }
837840

838841
public:
839-
PCHGenerator(const Preprocessor &PP, StringRef OutputFile,
842+
PCHGenerator(const Preprocessor &PP,
843+
StringRef OutputFile,
840844
clang::Module *Module,
841-
StringRef isysroot, raw_ostream *Out,
845+
StringRef isysroot,
842846
bool AllowASTWithErrors = false);
843847
~PCHGenerator();
844848
void InitializeSema(Sema &S) override { SemaPtr = &S; }
845849
void HandleTranslationUnit(ASTContext &Ctx) override;
846850
ASTMutationListener *GetASTMutationListener() override;
847851
ASTDeserializationListener *GetASTDeserializationListener() override;
848-
852+
/// \brief Register a callback to be invoked when the serialization is done.
853+
void RegisterSerializationFinishedCallback(
854+
const std::function<void(SmallVectorImpl<char>*)> Fn) {
855+
SerializationFinishedCallback = Fn;
856+
}
849857
bool hasEmittedPCH() const { return HasEmittedPCH; }
850858
};
851859

lib/CodeGen/CMakeLists.txt

+2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
set(LLVM_LINK_COMPONENTS
2+
${LLVM_TARGETS_TO_BUILD}
23
Analysis
34
BitReader
45
BitWriter
@@ -63,6 +64,7 @@ add_clang_library(clangCodeGen
6364
CodeGenAction.cpp
6465
CodeGenFunction.cpp
6566
CodeGenModule.cpp
67+
CodeGenModuleContainer.cpp
6668
CodeGenPGO.cpp
6769
CodeGenTBAA.cpp
6870
CodeGenTypes.cpp
+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
//===--- CodeGenModuleContainer.cpp - Emit .pcm files ---------------------===//
2+
//
3+
// The LLVM Compiler Infrastructure
4+
//
5+
// This file is distributed under the University of Illinois Open Source
6+
// License. See LICENSE.TXT for details.
7+
//
8+
//===----------------------------------------------------------------------===//
9+
10+
#include "clang/CodeGen/CodeGenModuleContainer.h"
11+
#include "CodeGenModule.h"
12+
#include "clang/AST/ASTContext.h"
13+
#include "clang/AST/DeclObjC.h"
14+
#include "clang/AST/Expr.h"
15+
#include "clang/AST/RecursiveASTVisitor.h"
16+
#include "clang/Basic/Diagnostic.h"
17+
#include "clang/Basic/TargetInfo.h"
18+
#include "clang/CodeGen/BackendUtil.h"
19+
#include "clang/Frontend/CodeGenOptions.h"
20+
#include "clang/Serialization/ASTWriter.h"
21+
#include "llvm/ADT/StringRef.h"
22+
#include "llvm/IR/Constants.h"
23+
#include "llvm/IR/DataLayout.h"
24+
#include "llvm/IR/LLVMContext.h"
25+
#include "llvm/IR/Module.h"
26+
#include "llvm/Support/TargetRegistry.h"
27+
#include <memory>
28+
using namespace clang;
29+
30+
namespace {
31+
class ModuleContainerGenerator : public CodeGenerator {
32+
DiagnosticsEngine &Diags;
33+
std::unique_ptr<const llvm::DataLayout> TD;
34+
ASTContext *Ctx;
35+
const CodeGenOptions CodeGenOpts;
36+
const TargetOptions TargetOpts;
37+
const LangOptions LangOpts;
38+
llvm::LLVMContext VMContext;
39+
std::unique_ptr<llvm::Module> M;
40+
std::unique_ptr<CodeGen::CodeGenModule> Builder;
41+
raw_ostream *OS;
42+
SmallVectorImpl<char> *SerializedASTBuffer;
43+
44+
public:
45+
ModuleContainerGenerator(DiagnosticsEngine &diags,
46+
const std::string &ModuleName,
47+
const CodeGenOptions &CGO, const TargetOptions &TO,
48+
const LangOptions &LO, raw_ostream *OS,
49+
PCHGenerator *PCHGen)
50+
: Diags(diags), CodeGenOpts(CGO), TargetOpts(TO), LangOpts(LO),
51+
M(new llvm::Module(ModuleName, VMContext)), OS(OS) {
52+
PCHGen->RegisterSerializationFinishedCallback(
53+
[&](SmallVectorImpl<char> *Buf){
54+
SerializedASTBuffer = Buf;
55+
});
56+
}
57+
58+
virtual ~ModuleContainerGenerator() {}
59+
llvm::Module *GetModule() override { return M.get(); }
60+
llvm::Module *ReleaseModule() override { return M.release(); }
61+
62+
/// Lifted from ModuleBuilder.
63+
const Decl *GetDeclForMangledName(StringRef MangledName) override {
64+
GlobalDecl Result;
65+
if (!Builder->lookupRepresentativeDecl(MangledName, Result))
66+
return nullptr;
67+
const Decl *D = Result.getCanonicalDecl().getDecl();
68+
if (auto FD = dyn_cast<FunctionDecl>(D)) {
69+
if (FD->hasBody(FD))
70+
return FD;
71+
} else if (auto TD = dyn_cast<TagDecl>(D)) {
72+
if (auto Def = TD->getDefinition())
73+
return Def;
74+
}
75+
return D;
76+
}
77+
78+
void Initialize(ASTContext &Context) override {
79+
Ctx = &Context;
80+
M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple());
81+
M->setDataLayout(Ctx->getTargetInfo().getTargetDescription());
82+
TD.reset(new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription()));
83+
Builder.reset(
84+
new CodeGen::CodeGenModule(Context, CodeGenOpts, *M, *TD, Diags));
85+
}
86+
87+
/// Emit a container holding the serialized AST.
88+
void HandleTranslationUnit(ASTContext &Ctx) override {
89+
if (Diags.hasErrorOccurred()) {
90+
if (Builder)
91+
Builder->clear();
92+
M.reset();
93+
return;
94+
}
95+
96+
// Finalize the Builder.
97+
if (Builder)
98+
Builder->Release();
99+
100+
// Initialize the backend if we haven't done so already.
101+
LLVMInitializeAllTargetInfos();
102+
LLVMInitializeAllTargets();
103+
LLVMInitializeAllAsmPrinters();
104+
LLVMInitializeAllTargetMCs();
105+
106+
// Ensure the target exists.
107+
std::string Error;
108+
auto Triple = Ctx.getTargetInfo().getTriple();
109+
if (!llvm::TargetRegistry::lookupTarget(Triple.getTriple(), Error))
110+
llvm::report_fatal_error(Error);
111+
112+
// Emit the serialized Clang AST into its own section.
113+
auto Size = SerializedASTBuffer->size();
114+
auto Int8Ty = llvm::Type::getInt8Ty(VMContext);
115+
auto *Ty = llvm::ArrayType::get(Int8Ty, Size);
116+
auto *Data = llvm::ConstantDataArray::getString(VMContext,
117+
StringRef(SerializedASTBuffer->data(), Size), /*AddNull=*/false);
118+
auto *ASTSym = new llvm::GlobalVariable(*M, Ty, /*constant*/ true,
119+
llvm::GlobalVariable::InternalLinkage, Data, "__clang_ast");
120+
ASTSym->setAlignment(8);
121+
if (Triple.isOSBinFormatMachO())
122+
// Include Mach-O segment name.
123+
ASTSym->setSection("__CLANG,__clangast");
124+
else if (Triple.isOSBinFormatCOFF())
125+
// Adhere to COFF eight-character limit.
126+
ASTSym->setSection("clangast");
127+
else
128+
ASTSym->setSection("__clangast");
129+
130+
// Use the LLVM backend to emit the pcm.
131+
clang::EmitBackendOutput(Diags, CodeGenOpts, TargetOpts, LangOpts,
132+
Ctx.getTargetInfo().getTargetDescription(), M.get(),
133+
BackendAction::Backend_EmitObj, OS);
134+
135+
// Make sure the module container hits disk now.
136+
OS->flush();
137+
138+
// Free up some memory, in case the process is kept alive.
139+
SerializedASTBuffer->clear();
140+
}
141+
};
142+
}
143+
144+
CodeGenerator *clang::CreateModuleContainerGenerator(
145+
DiagnosticsEngine &Diags, const std::string &ModuleName,
146+
const CodeGenOptions &CGO, const TargetOptions &TO, const LangOptions &LO,
147+
llvm::raw_ostream *OS, PCHGenerator *PCHGen) {
148+
return
149+
new ModuleContainerGenerator(Diags, ModuleName, CGO, TO, LO, OS, PCHGen);
150+
}

lib/Frontend/ASTUnit.cpp

+16-2
Original file line numberDiff line numberDiff line change
@@ -914,13 +914,20 @@ class PrecompilePreambleConsumer : public PCHGenerator {
914914
unsigned &Hash;
915915
std::vector<Decl *> TopLevelDecls;
916916
PrecompilePreambleAction *Action;
917+
raw_ostream *Out;
918+
SmallVectorImpl<char> *SerializedASTBuffer;
917919

918920
public:
919921
PrecompilePreambleConsumer(ASTUnit &Unit, PrecompilePreambleAction *Action,
920922
const Preprocessor &PP, StringRef isysroot,
921923
raw_ostream *Out)
922-
: PCHGenerator(PP, "", nullptr, isysroot, Out, /*AllowASTWithErrors=*/true),
923-
Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action) {
924+
: PCHGenerator(PP, "", nullptr, isysroot, /*AllowASTWithErrors=*/true),
925+
Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action),
926+
Out(Out) {
927+
RegisterSerializationFinishedCallback(
928+
[&](SmallVectorImpl<char> *Buf){
929+
SerializedASTBuffer = Buf;
930+
});
924931
Hash = 0;
925932
}
926933

@@ -941,6 +948,13 @@ class PrecompilePreambleConsumer : public PCHGenerator {
941948
void HandleTranslationUnit(ASTContext &Ctx) override {
942949
PCHGenerator::HandleTranslationUnit(Ctx);
943950
if (hasEmittedPCH()) {
951+
// Write the generated bitstream to "Out".
952+
Out->write((char *)&SerializedASTBuffer->front(),
953+
SerializedASTBuffer->size());
954+
// Make sure it hits disk now.
955+
Out->flush();
956+
SerializedASTBuffer->clear();
957+
944958
// Translate the top-level declarations we captured during
945959
// parsing into declaration IDs in the precompiled
946960
// preamble. This will allow us to deserialize those top-level

lib/Frontend/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ add_clang_library(clangFrontend
4545
LINK_LIBS
4646
clangAST
4747
clangBasic
48+
clangCodeGen
4849
clangDriver
4950
clangEdit
5051
clangLex

lib/Frontend/ChainedIncludesSource.cpp

+10-6
Original file line numberDiff line numberDiff line change
@@ -156,11 +156,13 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
156156
&Clang->getPreprocessor());
157157
Clang->createASTContext();
158158

159-
SmallVector<char, 256> serialAST;
160-
llvm::raw_svector_ostream OS(serialAST);
161-
auto consumer =
162-
llvm::make_unique<PCHGenerator>(Clang->getPreprocessor(), "-", nullptr,
163-
/*isysroot=*/"", &OS);
159+
auto consumer = llvm::make_unique<PCHGenerator>(Clang->getPreprocessor(),
160+
"-", nullptr, /*isysroot=*/"");
161+
SmallVectorImpl<char> *serialAST;
162+
consumer->RegisterSerializationFinishedCallback(
163+
[&](SmallVectorImpl<char> *Buf){
164+
serialAST = Buf;
165+
});
164166
Clang->getASTContext().setASTMutationListener(
165167
consumer->GetASTMutationListener());
166168
Clang->setASTConsumer(std::move(consumer));
@@ -197,7 +199,9 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
197199

198200
ParseAST(Clang->getSema());
199201
Clang->getDiagnosticClient().EndSourceFile();
200-
SerialBufs.push_back(llvm::MemoryBuffer::getMemBufferCopy(OS.str()));
202+
SerialBufs.push_back(llvm::MemoryBuffer::
203+
getMemBufferCopy(StringRef(serialAST->data(), serialAST->size())));
204+
serialAST->clear();
201205
source->CIs.push_back(Clang.release());
202206
}
203207

0 commit comments

Comments
 (0)