[WPD/VFE] Always emit vcall_visibility metadata for -fwhole-program-vtables

teresajohnson · teresajohnson · commit 458676db6e41 · 2020-01-23T11:36:01.000-08:00
Summary: First patch to support Safe Whole Program Devirtualization Enablement, see RFC here: http://lists.llvm.org/pipermail/llvm-dev/2019-December/137543.html Always emit !vcall_visibility metadata under -fwhole-program-vtables, and not just for -fvirtual-function-elimination. The vcall visibility metadata will (in a subsequent patch) be used to communicate to WPD which vtables are safe to devirtualize, and we will optionally convert the metadata to hidden visibility at link time. Subsequent follow on patches will help enable this by adding vcall_visibility metadata to the ThinLTO summaries, and always emit type test intrinsics under -fwhole-program-vtables (and not just for vtables with hidden visibility). In order to do this safely with VFE, since for VFE all vtable loads must be type checked loads which will no longer be the case, this patch adds a new "Virtual Function Elim" module flag to communicate to GlobalDCE whether to perform VFE using the vcall_visibility metadata. One additional advantage of using the vcall_visibility metadata to drive more WPD at LTO link time is that we can use the same mechanism to enable more aggressive VFE at LTO link time as well. The link time option proposed in the RFC will convert vcall_visibility metadata to hidden (aka linkage unit visibility), which combined with -fvirtual-function-elimination will allow it to be done more aggressively at LTO link time under the same conditions. Reviewers: pcc, ostannard, evgeny777, steven_wu Subscribers: mehdi_amini, Prazek, hiraditya, dexonsmith, davidxl, cfe-commits, llvm-commits Tags: #clang, #llvm Differential Revision: https://reviews.llvm.org/D71907
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
@@ -1131,9 +1131,10 @@ void CodeGenModule::EmitVTableTypeMetadata(const CXXRecordDecl *RD,
     }
   }
 
-  if (getCodeGenOpts().VirtualFunctionElimination) {
+  if (getCodeGenOpts().VirtualFunctionElimination ||
+      getCodeGenOpts().WholeProgramVTables) {
     llvm::GlobalObject::VCallVisibility TypeVis = GetVCallVisibilityLevel(RD);
     if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic)
-      VTable->addVCallVisibilityMetadata(TypeVis);
+      VTable->setVCallVisibilityMetadata(TypeVis);
   }
 }
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -549,6 +549,14 @@ void CodeGenModule::Release() {
     getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1);
   }
 
+  if (CodeGenOpts.WholeProgramVTables) {
+    // Indicate whether VFE was enabled for this module, so that the
+    // vcall_visibility metadata added under whole program vtables is handled
+    // appropriately in the optimizer.
+    getModule().addModuleFlag(llvm::Module::Error, "Virtual Function Elim",
+                              CodeGenOpts.VirtualFunctionElimination);
+  }
+
   if (LangOpts.Sanitize.has(SanitizerKind::CFIICall)) {
     getModule().addModuleFlag(llvm::Module::Override,
                               "CFI Canonical Jump Tables",
diff --git a/clang/test/CodeGenCXX/vcall-visibility-metadata.cpp b/clang/test/CodeGenCXX/vcall-visibility-metadata.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -emit-llvm -fvirtual-function-elimination -fwhole-program-vtables -o - %s | FileCheck %s
+// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -emit-llvm -fvirtual-function-elimination -fwhole-program-vtables -o - %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-VFE
+// RUN: %clang_cc1 -flto -flto-unit -triple x86_64-unknown-linux -emit-llvm -fwhole-program-vtables -o - %s | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOVFE
 
 
 // Anonymous namespace.
@@ -83,6 +84,7 @@ void *construct_G() {
   return new G();
 }
 
-
 // CHECK-DAG: [[VIS_DSO]] = !{i64 1}
 // CHECK-DAG: [[VIS_TU]] = !{i64 2}
+// CHECK-VFE-DAG: !{i32 1, !"Virtual Function Elim", i32 1}
+// CHECK-NOVFE-DAG: !{i32 1, !"Virtual Function Elim", i32 0}
diff --git a/llvm/include/llvm/IR/GlobalObject.h b/llvm/include/llvm/IR/GlobalObject.h
@@ -183,7 +183,7 @@ class GlobalObject : public GlobalValue {
   void copyMetadata(const GlobalObject *Src, unsigned Offset);
 
   void addTypeMetadata(unsigned Offset, Metadata *TypeID);
-  void addVCallVisibilityMetadata(VCallVisibility Visibility);
+  void setVCallVisibilityMetadata(VCallVisibility Visibility);
   VCallVisibility getVCallVisibility() const;
 
 protected:
diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp
@@ -1500,7 +1500,10 @@ void GlobalObject::addTypeMetadata(unsigned Offset, Metadata *TypeID) {
                      TypeID}));
 }
 
-void GlobalObject::addVCallVisibilityMetadata(VCallVisibility Visibility) {
+void GlobalObject::setVCallVisibilityMetadata(VCallVisibility Visibility) {
+  // Remove any existing vcall visibility metadata first in case we are
+  // updating.
+  eraseMetadata(LLVMContext::MD_vcall_visibility);
   addMetadata(LLVMContext::MD_vcall_visibility,
               *MDNode::get(getContext(),
                            {ConstantAsMetadata::get(ConstantInt::get(
diff --git a/llvm/lib/Transforms/IPO/GlobalDCE.cpp b/llvm/lib/Transforms/IPO/GlobalDCE.cpp
@@ -263,6 +263,15 @@ void GlobalDCEPass::AddVirtualFunctionDependencies(Module &M) {
   if (!ClEnableVFE)
     return;
 
+  // If the Virtual Function Elim module flag is present and set to zero, then
+  // the vcall_visibility metadata was inserted for another optimization (WPD)
+  // and we may not have type checked loads on all accesses to the vtable.
+  // Don't attempt VFE in that case.
+  auto *Val = mdconst::dyn_extract_or_null<ConstantInt>(
+      M.getModuleFlag("Virtual Function Elim"));
+  if (!Val || Val->getZExtValue() == 0)
+    return;
+
   ScanVTables(M);
 
   if (VFESafeVTables.empty())
diff --git a/llvm/lib/Transforms/IPO/GlobalSplit.cpp b/llvm/lib/Transforms/IPO/GlobalSplit.cpp
@@ -111,6 +111,9 @@ static bool splitGlobal(GlobalVariable &GV) {
                             ConstantInt::get(Int32Ty, ByteOffset - SplitBegin)),
                         Type->getOperand(1)}));
     }
+
+    if (GV.hasMetadata(LLVMContext::MD_vcall_visibility))
+      SplitGV->setVCallVisibilityMetadata(GV.getVCallVisibility());
   }
 
   for (User *U : GV.users()) {
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-call.ll
@@ -70,9 +70,12 @@ entry:
 
 declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) #2
 
+!llvm.module.flags = !{!5}
+
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
 !2 = !{i64 2}
 !3 = !{i64 16, !"_ZTS1B"}
 !4 = !{i64 16, !"_ZTSM1BFivE.virtual"}
+!5 = !{i32 1, !"Virtual Function Elim", i32 1}
 !10 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-base-pointer-call.ll
@@ -108,11 +108,14 @@ memptr.end:                                       ; preds = %memptr.nonvirtual,
 
 declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
 
+!llvm.module.flags = !{!7}
+
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFiiE.virtual"}
 !2 = !{i64 24, !"_ZTSM1AFifE.virtual"}
 !3 = !{i64 2}
 !4 = !{i64 16, !"_ZTS1B"}
 !5 = !{i64 16, !"_ZTSM1BFiiE.virtual"}
 !6 = !{i64 24, !"_ZTSM1BFifE.virtual"}
+!7 = !{i32 1, !"Virtual Function Elim", i32 1}
 !12 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-call.ll
@@ -70,9 +70,12 @@ entry:
 
 declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata) #2
 
+!llvm.module.flags = !{!5}
+
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
 !2 = !{i64 2}
 !3 = !{i64 16, !"_ZTS1B"}
 !4 = !{i64 16, !"_ZTSM1BFivE.virtual"}
+!5 = !{i32 1, !"Virtual Function Elim", i32 1}
 !10 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-derived-pointer-call.ll
@@ -110,11 +110,14 @@ memptr.end:                                       ; preds = %memptr.nonvirtual,
 
 declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
 
+!llvm.module.flags = !{!7}
+
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFiiE.virtual"}
 !2 = !{i64 24, !"_ZTSM1AFifE.virtual"}
 !3 = !{i64 2}
 !4 = !{i64 16, !"_ZTS1B"}
 !5 = !{i64 16, !"_ZTSM1BFiiE.virtual"}
 !6 = !{i64 24, !"_ZTSM1BFifE.virtual"}
+!7 = !{i32 1, !"Virtual Function Elim", i32 1}
 !12 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-novfe.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-novfe.ll
@@ -0,0 +1,59 @@
+; Tests that VFE is not performed when the Virtual Function Elim metadata set
+; to 0. This is the same as virtual-functions.ll otherwise.
+; RUN: opt < %s -globaldce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare dso_local noalias nonnull i8* @_Znwm(i64)
+declare { i8*, i1 } @llvm.type.checked.load(i8*, i32, metadata)
+
+; %struct.A is a C++ struct with two virtual functions, A::foo and A::bar. The
+; !vcall_visibility metadata is set on the vtable, so we know that all virtual
+; calls through this vtable are visible and use the @llvm.type.checked.load
+; intrinsic. Function test_A makes a call to A::foo, but there is no call to
+; A::bar anywhere, so A::bar can be deleted, and its vtable slot replaced with
+; null.
+; However, with the metadata set to 0 we should not perform this VFE.
+
+%struct.A = type { i32 (...)** }
+
+; We should retain @_ZN1A3barEv in the vtable.
+; CHECK: @_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)] }
+@_ZTV1A = internal unnamed_addr constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* null, i8* bitcast (i32 (%struct.A*)* @_ZN1A3fooEv to i8*), i8* bitcast (i32 (%struct.A*)* @_ZN1A3barEv to i8*)] }, align 8, !type !0, !type !1, !type !2, !vcall_visibility !3
+
+; A::foo is called, so must be retained.
+; CHECK: define internal i32 @_ZN1A3fooEv(
+define internal i32 @_ZN1A3fooEv(%struct.A* nocapture readnone %this) {
+entry:
+  ret i32 42
+}
+
+; A::bar is not used, so can be deleted with VFE, however, we should not be
+; performing that elimination here.
+; CHECK: define internal i32 @_ZN1A3barEv(
+define internal i32 @_ZN1A3barEv(%struct.A* nocapture readnone %this) {
+entry:
+  ret i32 1337
+}
+
+define dso_local i32 @test_A() {
+entry:
+  %call = tail call i8* @_Znwm(i64 8)
+  %0 = bitcast i8* %call to %struct.A*
+  %1 = bitcast i8* %call to i32 (...)***
+  store i32 (...)** bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i32 (...)**), i32 (...)*** %1, align 8
+  %2 = tail call { i8*, i1 } @llvm.type.checked.load(i8* bitcast (i8** getelementptr inbounds ({ [4 x i8*] }, { [4 x i8*] }* @_ZTV1A, i64 0, inrange i32 0, i64 2) to i8*), i32 0, metadata !"_ZTS1A"), !nosanitize !9
+  %3 = extractvalue { i8*, i1 } %2, 0, !nosanitize !9
+  %4 = bitcast i8* %3 to i32 (%struct.A*)*, !nosanitize !9
+  %call1 = tail call i32 %4(%struct.A* nonnull %0)
+  ret i32 %call1
+}
+
+!llvm.module.flags = !{!4}
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
+!2 = !{i64 24, !"_ZTSM1AFivE.virtual"}
+!3 = !{i64 2}
+!4 = !{i32 1, !"Virtual Function Elim", i32 0}
+!9 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-post-lto.ll
@@ -85,11 +85,12 @@ entry:
 
 declare dso_local noalias nonnull i8* @_Znwm(i64)
 
-!llvm.module.flags = !{!5}
+!llvm.module.flags = !{!5, !6}
 
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
 !2 = !{i64 0} ; public vcall visibility
 !3 = !{i64 1} ; linkage-unit vcall visibility
 !4 = !{i64 2} ; translation-unit vcall visibility
 !5 = !{i32 1, !"LTOPostLink", i32 1}
+!6 = !{i32 1, !"Virtual Function Elim", i32 1}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-visibility-pre-lto.ll
@@ -85,10 +85,11 @@ entry:
 
 declare dso_local noalias nonnull i8* @_Znwm(i64)
 
-!llvm.module.flags = !{}
+!llvm.module.flags = !{!5}
 
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
 !2 = !{i64 0} ; public vcall visibility
 !3 = !{i64 1} ; linkage-unit vcall visibility
 !4 = !{i64 2} ; translation-unit vcall visibility
+!5 = !{i32 1, !"Virtual Function Elim", i32 1}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions.ll
@@ -48,8 +48,11 @@ entry:
   ret i32 %call1
 }
 
+!llvm.module.flags = !{!4}
+
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFivE.virtual"}
 !2 = !{i64 24, !"_ZTSM1AFivE.virtual"}
 !3 = !{i64 2}
+!4 = !{i32 1, !"Virtual Function Elim", i32 1}
 !9 = !{}
diff --git a/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll b/llvm/test/Transforms/GlobalDCE/vtable-rtti.ll
@@ -39,9 +39,10 @@ entry:
 declare dso_local noalias nonnull i8* @_Znwm(i64)
 @_ZTVN10__cxxabiv117__class_type_infoE = external dso_local global i8*
 
-!llvm.module.flags = !{!3}
+!llvm.module.flags = !{!3, !4}
 
 !0 = !{i64 16, !"_ZTS1A"}
 !1 = !{i64 16, !"_ZTSM1AFvvE.virtual"}
 !2 = !{i64 2} ; translation-unit vcall visibility
 !3 = !{i32 1, !"LTOPostLink", i32 1}
+!4 = !{i32 1, !"Virtual Function Elim", i32 1}
diff --git a/llvm/test/Transforms/GlobalSplit/basic.ll b/llvm/test/Transforms/GlobalSplit/basic.ll
@@ -12,13 +12,13 @@ target triple = "x86_64-unknown-linux-gnu"
 ]
 
 ; CHECK-NOT: @global =
-; CHECK: @global.0 = private constant [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2], !type [[T1:![0-9]+]], !type [[T2:![0-9]+]], !type [[T3:![0-9]+$]]
-; CHECK: @global.1 = private constant [1 x i8* ()*] [i8* ()* @f3], !type [[T4:![0-9]+]], !type [[T5:![0-9]+$]]
+; CHECK: @global.0 = private constant [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2], !type [[T1:![0-9]+]], !type [[T2:![0-9]+]], !type [[T3:![0-9]+]], !vcall_visibility [[VIS:![0-9]+$]]
+; CHECK: @global.1 = private constant [1 x i8* ()*] [i8* ()* @f3], !type [[T4:![0-9]+]], !type [[T5:![0-9]+]], !vcall_visibility [[VIS$]]
 ; CHECK-NOT: @global =
 @global = internal constant { [2 x i8* ()*], [1 x i8* ()*] } {
   [2 x i8* ()*] [i8* ()* @f1, i8* ()* @f2],
   [1 x i8* ()*] [i8* ()* @f3]
-}, !type !0, !type !1, !type !2, !type !3, !type !4
+}, !type !0, !type !1, !type !2, !type !3, !type !4, !vcall_visibility !5
 
 ; CHECK: define i8* @f1()
 define i8* @f1() {
@@ -54,10 +54,12 @@ declare i1 @llvm.type.test(i8*, metadata) nounwind readnone
 ; CHECK: [[T1]] = !{i32 0, !"foo"}
 ; CHECK: [[T2]] = !{i32 15, !"bar"}
 ; CHECK: [[T3]] = !{i32 16, !"a"}
+; CHECK: [[VIS]] = !{i64 2}
 ; CHECK: [[T4]] = !{i32 1, !"b"}
 ; CHECK: [[T5]] = !{i32 8, !"c"}
 !0 = !{i32 0, !"foo"}
 !1 = !{i32 15, !"bar"}
 !2 = !{i32 16, !"a"}
 !3 = !{i32 17, !"b"}
 !4 = !{i32 24, !"c"}
+!5 = !{i64 2}

Original file line number	Diff line number	Diff line change
`@@ -1131,9 +1131,10 @@ void CodeGenModule::EmitVTableTypeMetadata(const CXXRecordDecl *RD,`
`1131`	`1131`	`}`
`1132`	`1132`	`}`
`1133`	`1133`
`1134`		`- if (getCodeGenOpts().VirtualFunctionElimination) {`
	`1134`	`+ if (getCodeGenOpts().VirtualFunctionElimination \|\|`
	`1135`	`+ getCodeGenOpts().WholeProgramVTables) {`
`1135`	`1136`	`llvm::GlobalObject::VCallVisibility TypeVis = GetVCallVisibilityLevel(RD);`
`1136`	`1137`	`if (TypeVis != llvm::GlobalObject::VCallVisibilityPublic)`
`1137`		`- VTable->addVCallVisibilityMetadata(TypeVis);`
	`1138`	`+ VTable->setVCallVisibilityMetadata(TypeVis);`
`1138`	`1139`	`}`
`1139`	`1140`	`}`
Original file line number	Diff line number	Diff line change
`@@ -111,6 +111,9 @@ static bool splitGlobal(GlobalVariable &GV) {`
`111`	`111`	`ConstantInt::get(Int32Ty, ByteOffset - SplitBegin)),`
`112`	`112`	`Type->getOperand(1)}));`
`113`	`113`	`}`
	`114`	`+`
	`115`	`+ if (GV.hasMetadata(LLVMContext::MD_vcall_visibility))`
	`116`	`+ SplitGV->setVCallVisibilityMetadata(GV.getVCallVisibility());`
`114`	`117`	`}`
`115`	`118`
`116`	`119`	`for (User *U : GV.users()) {`