Skip to content

Split up debug_abbrev and debug_info sections into multiple blocks #4778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Split up debug_abbrev and debug_info sections into multiple blocks #4778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b4b1103
Split up debug info and debug abbrev section in the flatv1 schema, an…
rastogishubham May 31, 2022
70a75ef
Nestedv1 also splits debug_abbrev and debug_info blocks properly
rastogishubham Jul 8, 2022
319abbc
Fix merge conflict issues
rastogishubham Jul 19, 2022
7d23851
Address changes requested in PR review by moving the debug_info and d…
rastogishubham Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion llvm/include/llvm/CASObjectFormats/Data.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@ class BlockData {
/// \p Fixups add no storage cost if they are not used.
static void encode(uint64_t Size, uint64_t Alignment,
uint64_t AlignmentOffset, Optional<StringRef> Content,
ArrayRef<Fixup> Fixups, SmallVectorImpl<char> &Data);
ArrayRef<Fixup> Fixups, SmallVectorImpl<char> &Data,
bool IsDebugInfoBlock = false);
Copy link

@cachemeifyoucan cachemeifyoucan Jul 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this belong here. The encode function here is simply the helper function how to encode a data block with fixups.

Debug Info Block should be a different block from block ref and the branch to select how to encode debug info block should be above this function:

if (IsDebugInfoBlock)
   DebugInfoBlock::encode(...)
else
   BlockData::encode(...)


Error decode(uint64_t &Size, uint64_t &Alignment, uint64_t &AlignmentOffset,
Optional<StringRef> &Content, FixupList &Fixups) const;
Expand Down
6 changes: 5 additions & 1 deletion llvm/include/llvm/CASObjectFormats/FlatV1.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#define LLVM_CASOBJECTFORMATS_FLATV1_H

#include "llvm/CAS/CASID.h"
#include "llvm/CAS/CASReference.h"
#include "llvm/CASObjectFormats/CASObjectReader.h"
#include "llvm/CASObjectFormats/Data.h"
#include "llvm/CASObjectFormats/ObjectFormatSchemaBase.h"
Expand Down Expand Up @@ -208,7 +209,8 @@ class BlockRef : public SpecificRef<BlockRef> {
static constexpr StringLiteral KindString = "cas.o:block";

static Expected<BlockRef> create(CompileUnitBuilder &CUB,
const jitlink::Block &Block);
const jitlink::Block &Block,
cas::ObjectRef *AbbrevRef = nullptr);

static Expected<BlockRef> get(Expected<ObjectFormatObjectProxy> Ref);
static Expected<BlockRef> get(const ObjectFileSchema &Schema,
Expand Down Expand Up @@ -313,6 +315,8 @@ class CompileUnitBuilder {

Error createSection(const jitlink::Section &S);
Error createBlock(const jitlink::Block &B);
Expected<cas::ObjectRef> createAbbrevBlock(const jitlink::Block &B);
Error createInfoBlock(const jitlink::Block &B, cas::ObjectRef *AbbrevID);
Error createSymbol(const jitlink::Symbol &S);

// Lookup functions. The result must exist in the cache already.
Expand Down
31 changes: 17 additions & 14 deletions llvm/include/llvm/CASObjectFormats/NestedV1.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#define LLVM_CASOBJECTFORMATS_NESTEDV1_H

#include "llvm/CAS/CASID.h"
#include "llvm/CAS/CASReference.h"
#include "llvm/CASObjectFormats/Data.h"
#include "llvm/CASObjectFormats/ObjectFormatSchemaBase.h"
#include "llvm/ExecutionEngine/JITLink/JITLink.h"
Expand Down Expand Up @@ -273,11 +274,10 @@ class BlockDataRef : public SpecificRef<BlockDataRef> {
uint64_t AlignmentOffset,
ArrayRef<Fixup> Fixups);

static Expected<BlockDataRef> createContent(const ObjectFileSchema &Schema,
StringRef Content,
uint64_t Alignment,
uint64_t AlignmentOffset,
ArrayRef<Fixup> Fixups);
static Expected<BlockDataRef>
createContent(const ObjectFileSchema &Schema, StringRef Content,
uint64_t Alignment, uint64_t AlignmentOffset,
ArrayRef<Fixup> Fixups, bool IsDebugInfoBlock = false);

static Expected<BlockDataRef> create(const ObjectFileSchema &Schema,
const jitlink::Block &Block,
Expand All @@ -288,11 +288,10 @@ class BlockDataRef : public SpecificRef<BlockDataRef> {
explicit BlockDataRef(SpecificRefT Ref, data::BlockData Data)
: SpecificRefT(Ref), Data(Data) {}

static Expected<BlockDataRef> createImpl(const ObjectFileSchema &Schema,
Optional<StringRef> Content,
uint64_t Size, uint64_t Alignment,
uint64_t AlignmentOffset,
ArrayRef<Fixup> Fixups);
static Expected<BlockDataRef>
createImpl(const ObjectFileSchema &Schema, Optional<StringRef> Content,
uint64_t Size, uint64_t Alignment, uint64_t AlignmentOffset,
ArrayRef<Fixup> Fixups, bool IsDebugInfoBlock = false);
};

/// A variant of SymbolRef and IndirectSymbolRef. The kind is cached.
Expand Down Expand Up @@ -521,7 +520,8 @@ class BlockRef : public SpecificRef<BlockRef> {
static Expected<BlockRef>
create(const ObjectFileSchema &Schema, const jitlink::Block &Block,
function_ref<Expected<Optional<TargetRef>>(const jitlink::Symbol &)>
GetTargetRef);
GetTargetRef,
cas::ObjectRef *AbbrevRef = nullptr);

static Expected<BlockRef> create(const ObjectFileSchema &Schema,
SectionRef Section, BlockDataRef Data) {
Expand All @@ -535,7 +535,7 @@ class BlockRef : public SpecificRef<BlockRef> {
return createImpl(Schema, Section, Data, TargetInfo, Targets, Fixups);
}

static Expected<BlockRef> get(Expected<ObjectFormatObjectProxy> Ref);
static Expected<BlockRef> get(Expected<ObjectFormatObjectProxy> Ref, bool IsDebugInfoBlock = false);
static Expected<BlockRef> get(const ObjectFileSchema &Schema,
cas::ObjectRef ID) {
return get(Schema.get(ID));
Expand All @@ -562,7 +562,9 @@ class BlockRef : public SpecificRef<BlockRef> {
SectionRef Section, BlockDataRef Data,
ArrayRef<TargetInfo> TargetInfo,
ArrayRef<TargetRef> Targets,
ArrayRef<Fixup> Fixups);
ArrayRef<Fixup> Fixups,
cas::ObjectRef *AbbrevRef = nullptr,
bool IsDebugInfoBlock = false);
};

/// A symbol.
Expand Down Expand Up @@ -693,7 +695,8 @@ class SymbolRef : public SpecificRef<SymbolRef> {
static Expected<SymbolRef> create(const ObjectFileSchema &Schema,
Optional<NameRef> SymbolName,
SymbolDefinitionRef Definition,
uint64_t Offset, Flags F);
uint64_t Offset, Flags F,
bool IsDebugInfoSymbol = false);

static Expected<SymbolRef>
create(const ObjectFileSchema &Schema, const jitlink::Symbol &S,
Expand Down
47 changes: 44 additions & 3 deletions llvm/lib/CASObjectFormats/Data.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ void FixupList::iterator::decode(bool IsInit) {

void BlockData::encode(uint64_t Size, uint64_t Alignment,
uint64_t AlignmentOffset, Optional<StringRef> Content,
ArrayRef<Fixup> Fixups, SmallVectorImpl<char> &Data) {
ArrayRef<Fixup> Fixups, SmallVectorImpl<char> &Data,
bool IsDebugInfoBlock) {
assert(!Content || Size == Content->size() && "Mismatched content size");
assert(Alignment && "Expected non-zero alignment");
assert(isPowerOf2_64(Alignment) && "Expected alignment to be a power of 2");
Expand All @@ -81,8 +82,48 @@ void BlockData::encode(uint64_t Size, uint64_t Alignment,
(unsigned(IsZeroFill) << IsZeroFillBit));

encoding::writeVBR8(Size, Data);
if (Content)
Data.append(Content->begin(), Content->end());
if (Content) {
if (!IsDebugInfoBlock) {
Data.append(Content->begin(), Content->end());
} else {
// This is a debug info block, the abbreviation offset can cause
// deduplication to fail, handle it here.
StringRef SizeOfUnit = Content->substr(0, 4);
unsigned SizeOfUnitInt = 0;
SizeOfUnitInt |= SizeOfUnit[3];
SizeOfUnitInt <<= 8;
SizeOfUnitInt |= SizeOfUnit[2];
SizeOfUnitInt <<= 8;
SizeOfUnitInt |= SizeOfUnit[1];
SizeOfUnitInt <<= 8;
SizeOfUnitInt |= SizeOfUnit[0];
if (SizeOfUnitInt != 0xffffffff) {
// This is a 32-bit DWARF format, copy the first 6 bytes for size and
// DWARF version, skip the next 4 bytes and copy 0's, then copy the
// rest.
Data.append(Content->begin(), Content->begin() + 6);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.append(Content->begin() + 10, Content->end());
} else {
// This is a 64-bit DWARF format, copy the first 16 bytes for size,
// DWARF version, unit type, and address size, skip the next 8 bytes and
// copy 0's, then copy the rest.
Data.append(Content->begin(), Content->begin() + 16);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.push_back(0);
Data.append(Content->begin() + 24, Content->end());
}
}
}
FixupList::encode(Fixups, Data);

if (!HasAlignmentOffset)
Expand Down
79 changes: 73 additions & 6 deletions llvm/lib/CASObjectFormats/FlatV1.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/CAS/CASDB.h"
#include "llvm/CAS/CASReference.h"
#include "llvm/CASObjectFormats/Data.h"
#include "llvm/CASObjectFormats/Encoding.h"
#include "llvm/CASObjectFormats/ObjectFormatHelpers.h"
Expand Down Expand Up @@ -321,11 +322,20 @@ static Error decodeFixup(const FlatV1ObjectReader &Reader, StringRef &Data,
}

Expected<BlockRef> BlockRef::create(CompileUnitBuilder &CUB,
const jitlink::Block &Block) {
const jitlink::Block &Block,
cas::ObjectRef *AbbrevRef) {
Expected<Builder> B = Builder::startNode(CUB.Schema, KindString);
if (!B)
return B.takeError();

// If we are creating a cas block out of a debug_info jitlink::Block, add the
// debug_abbrev cas block CAS ID as a refrence
if (Block.getSection().getName() == "__DWARF,__debug_info") {
assert(AbbrevRef &&
"The CAS Ref for the abbrev section shouldn't be nullptr");
B->Refs.push_back(*AbbrevRef);
}

// Encode Section.
auto SectionIndex = CUB.getSectionIndex(Block.getSection());
if (!SectionIndex)
Expand Down Expand Up @@ -366,11 +376,11 @@ Expected<BlockRef> BlockRef::create(CompileUnitBuilder &CUB,
return std::move(E);
BlockSize = Content->size();
}

SmallString<1024> EncodeContent;
data::BlockData::encode(BlockSize, Block.getAlignment(),
Block.getAlignmentOffset(), Content, Fixups,
EncodeContent);
data::BlockData::encode(
BlockSize, Block.getAlignment(), Block.getAlignmentOffset(), Content,
Fixups, EncodeContent,
Block.getSection().getName() == "__DWARF,__debug_info");

StringRef BlockData(EncodeContent);
// Encode content first with size and data.
Expand Down Expand Up @@ -689,12 +699,37 @@ Error CompileUnitBuilder::createSection(const jitlink::Section &S) {
}

Error CompileUnitBuilder::createBlock(const jitlink::Block &B) {
if (B.getSection().getName() == "__DWARF,__debug_abbrev" ||
B.getSection().getName() == "__DWARF,__debug_info")
return Error::success();
// Store the current idx. It is created in order so just add in the end.
BlockIndexStarts.push_back(Indexes.size());
auto Block = BlockRef::create(*this, B);
if (!Block)
return Block.takeError();
commitNode(*Block);
return Error::success();
}

Expected<cas::ObjectRef>
CompileUnitBuilder::createAbbrevBlock(const jitlink::Block &B) {
// Store the current idx. It is created in order so just add in the end.
BlockIndexStarts.push_back(Indexes.size());
auto Block = BlockRef::create(*this, B);
if (!Block)
return Block.takeError();
commitNode(*Block);
return Block->getRef();
}

Error CompileUnitBuilder::createInfoBlock(const jitlink::Block &B,
cas::ObjectRef *AbbrevID) {
// Store the current idx. It is created in order so just add in the end.
BlockIndexStarts.push_back(Indexes.size());
auto Block = BlockRef::create(*this, B, AbbrevID);
if (!Block)
return Block.takeError();
commitNode(*Block);
return Error::success();
}

Expand Down Expand Up @@ -792,8 +827,24 @@ Expected<CompileUnitRef> CompileUnitRef::create(const ObjectFileSchema &Schema,
llvm::sort(Builder.Blocks.begin() + PreviousSize, Builder.Blocks.end(),
compareBlocksByAddress);
};
for (const jitlink::Section &Section : G.sections())
SmallVector<const jitlink::Block *, 16> AbbrevBlocks;
SmallVector<const jitlink::Block *, 16> InfoBlocks;
for (const jitlink::Section &Section : G.sections()) {
if (Section.getName() == "__DWARF,__debug_abbrev")
AbbrevBlocks.append(Section.blocks().begin(), Section.blocks().end());
else if (Section.getName() == "__DWARF,__debug_info")
InfoBlocks.append(Section.blocks().begin(), Section.blocks().end());
appendBlocks(Section.blocks());
}
// If the number of debug_abbrev blocks are 1, we assume that the
// abbreviations could not be split up, and every debug_info compile unit
// should have a reference to the same abbreviation.
assert(AbbrevBlocks.size() == 1 ||
AbbrevBlocks.size() == InfoBlocks.size() &&
"The number of Abbreviation contributions should be equal to the "
"number of Compile Units");
llvm::sort(InfoBlocks.begin(), InfoBlocks.end(), compareBlocksByAddress);
llvm::sort(AbbrevBlocks.begin(), AbbrevBlocks.end(), compareBlocksByAddress);

#ifndef NDEBUG
for (auto *S : makeArrayRef(Symbols).slice(0, DefinedSymbolsSize))
Expand All @@ -815,6 +866,22 @@ Expected<CompileUnitRef> CompileUnitRef::create(const ObjectFileSchema &Schema,
return std::move(E);
}

// Create BlockRefs for Abbrevs and Compile Units
SmallVector<cas::ObjectRef, 16> AbbrevRefs;
for (auto *B : AbbrevBlocks) {
Expected<cas::ObjectRef> Ref = Builder.createAbbrevBlock(*B);
if (!Ref)
return Ref.takeError();
AbbrevRefs.push_back(*Ref);
}
unsigned I = 0;
for (auto *B : InfoBlocks) {
if (auto E = Builder.createInfoBlock(
*B, AbbrevRefs.size() == 1 ? &AbbrevRefs[0] : &AbbrevRefs[I]))
return std::move(E);
I++;
}

auto B = Builder::startRootNode(Schema, KindString);
if (!B)
return B.takeError();
Expand Down
Loading