diff --git a/compiler-rt/test/metadata/uar.cpp b/compiler-rt/test/metadata/uar.cpp index cbafe462c364..2c5537b815a9 100644 --- a/compiler-rt/test/metadata/uar.cpp +++ b/compiler-rt/test/metadata/uar.cpp @@ -2,7 +2,7 @@ // RUN: %clangxx %s -O1 -o %t -fexperimental-sanitize-metadata=covered,uar -fsanitize=address,signed-integer-overflow,alignment && %t | FileCheck %s // RUN: %clangxx %s -O1 -o %t -mcmodel=large -fexperimental-sanitize-metadata=covered,uar -fsanitize=address,signed-integer-overflow,alignment && %t | FileCheck %s -// CHECK: metadata add version 2 +// CHECK-DAG: metadata add version 2 __attribute__((noinline, not_tail_called)) void escape(const volatile void *p) { [[maybe_unused]] static const volatile void *sink; @@ -14,51 +14,51 @@ __attribute__((noinline, not_tail_called)) void use(int x) { sink += x; } -// CHECK: empty: features=0 stack_args=0 +// CHECK-DAG: empty: features=0 stack_args=0 void empty() {} -// CHECK: simple: features=0 stack_args=0 +// CHECK-DAG: simple: features=0 stack_args=0 int simple(int *data, int index) { return data[index + 1]; } -// CHECK: builtins: features=0 stack_args=0 +// CHECK-DAG: builtins: features=0 stack_args=0 int builtins() { int x = 0; __builtin_prefetch(&x); return x; } -// CHECK: ellipsis: features=0 stack_args=0 +// CHECK-DAG: ellipsis: features=0 stack_args=0 void ellipsis(const char *fmt, ...) { int x; escape(&x); } -// CHECK: non_empty_function: features=2 stack_args=0 +// CHECK-DAG: non_empty_function: features=2 stack_args=0 void non_empty_function() { int x; escape(&x); } -// CHECK: no_stack_args: features=2 stack_args=0 +// CHECK-DAG: no_stack_args: features=2 stack_args=0 void no_stack_args(long a0, long a1, long a2, long a3, long a4, long a5) { int x; escape(&x); } -// CHECK: stack_args: features=6 stack_args=16 +// CHECK-DAG: stack_args: features=6 stack_args=16 void stack_args(long a0, long a1, long a2, long a3, long a4, long a5, long a6) { int x; escape(&x); } -// CHECK: more_stack_args: features=6 stack_args=32 +// CHECK-DAG: more_stack_args: features=6 stack_args=32 void more_stack_args(long a0, long a1, long a2, long a3, long a4, long a5, long a6, long a7, long a8) { int x; escape(&x); } -// CHECK: struct_stack_args: features=6 stack_args=144 +// CHECK-DAG: struct_stack_args: features=6 stack_args=144 struct large { char x[131]; }; @@ -69,28 +69,28 @@ void struct_stack_args(large a) { __attribute__((noinline)) int tail_called(int x) { return x; } -// CHECK: with_tail_call: features=2 +// CHECK-DAG: with_tail_call: features=2 int with_tail_call(int x) { [[clang::musttail]] return tail_called(x); } __attribute__((noinline, noreturn)) int noreturn(int x) { __builtin_trap(); } -// CHECK: with_noreturn_tail_call: features=0 +// CHECK-DAG: with_noreturn_tail_call: features=0 int with_noreturn_tail_call(int x) { return noreturn(x); } -// CHECK: local_array: features=0 +// CHECK-DAG: local_array: features=0 void local_array(int x) { int data[10]; use(data[x]); } -// CHECK: local_alloca: features=0 +// CHECK-DAG: local_alloca: features=0 void local_alloca(int size, int i, int j) { volatile int *p = static_cast(__builtin_alloca(size)); p[i] = 0; use(p[j]); } -// CHECK: escaping_alloca: features=2 +// CHECK-DAG: escaping_alloca: features=2 void escaping_alloca(int size, int i) { volatile int *p = static_cast(__builtin_alloca(size)); escape(&p[i]); diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp index d50218aaa3b6..603a1565e48c 100644 --- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp +++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp @@ -82,6 +82,7 @@ STATISTIC(NumNoUnwind, "Number of functions marked as nounwind"); STATISTIC(NumNoFree, "Number of functions marked as nofree"); STATISTIC(NumWillReturn, "Number of functions marked as willreturn"); STATISTIC(NumNoSync, "Number of functions marked as nosync"); +STATISTIC(NumCold, "Number of functions marked as cold"); STATISTIC(NumThinLinkNoRecurse, "Number of functions marked as norecurse during thinlink"); @@ -1745,6 +1746,7 @@ static bool canReturn(Function &F) { return false; } + // Set the noreturn function attribute if possible. static void addNoReturnAttrs(const SCCNodeSet &SCCNodes, SmallSet &Changed) { @@ -1760,6 +1762,72 @@ static void addNoReturnAttrs(const SCCNodeSet &SCCNodes, } } +static bool +allBBPathsGoThroughCold(BasicBlock *BB, + SmallDenseMap &Visited) { + // If BB contains a cold callsite this path through the CG is cold. + // Ignore whether the instructions actually are guranteed to transfer + // execution. Divergent behavior is considered unlikely. + if (any_of(*BB, [](Instruction &I) { + if (auto *CB = dyn_cast(&I)) + return CB->hasFnAttr(Attribute::Cold); + return false; + })) { + Visited[BB] = true; + return true; + } + + auto Succs = successors(BB); + // We found a path that doesn't go through any cold callsite. + if (Succs.empty()) + return false; + + // We didn't find a cold callsite in this BB, so check that all successors + // contain a cold callsite (or that their successors do). + // Potential TODO: We could use static branch hints to assume certain + // successor paths are inherently cold, irrespective of if they contain a cold + // callsite. + for (auto *Succ : Succs) { + // Start with false, this is necessary to ensure we don't turn loops into + // cold. + auto R = Visited.try_emplace(Succ, false); + if (!R.second) { + if (R.first->second) + continue; + return false; + } + if (!allBBPathsGoThroughCold(Succ, Visited)) + return false; + Visited[Succ] = true; + } + + return true; +} + +static bool allPathsGoThroughCold(Function &F) { + SmallDenseMap Visited; + Visited[&F.front()] = false; + return allBBPathsGoThroughCold(&F.front(), Visited); +} + +// Set the cold function attribute if possible. +static void addColdAttrs(const SCCNodeSet &SCCNodes, + SmallSet &Changed) { + for (Function *F : SCCNodes) { + if (!F || !F->hasExactDefinition() || F->hasFnAttribute(Attribute::Naked) || + F->hasFnAttribute(Attribute::Cold) || F->hasFnAttribute(Attribute::Hot)) + continue; + + // Potential TODO: We could add attribute `cold` on functions with `coldcc`. + if (allPathsGoThroughCold(*F)) { + F->addFnAttr(Attribute::Cold); + ++NumCold; + Changed.insert(F); + continue; + } + } +} + static bool functionWillReturn(const Function &F) { // We can infer and propagate function attributes only when we know that the // definition we'll get at link time is *exactly* the definition we see now. @@ -1853,6 +1921,7 @@ deriveAttrsInPostOrder(ArrayRef Functions, AARGetterT &&AARGetter, addArgumentAttrs(Nodes.SCCNodes, Changed); inferConvergent(Nodes.SCCNodes, Changed); addNoReturnAttrs(Nodes.SCCNodes, Changed); + addColdAttrs(Nodes.SCCNodes, Changed); addWillReturn(Nodes.SCCNodes, Changed); addNoUndefAttrs(Nodes.SCCNodes, Changed); diff --git a/llvm/test/Transforms/FunctionAttrs/cold.ll b/llvm/test/Transforms/FunctionAttrs/cold.ll index 1fa8ae067979..a205fbda0621 100644 --- a/llvm/test/Transforms/FunctionAttrs/cold.ll +++ b/llvm/test/Transforms/FunctionAttrs/cold.ll @@ -54,14 +54,23 @@ while.body2: } define void @test_no_exit() { -; COMMON: Function Attrs: noreturn -; COMMON-LABEL: define void @test_no_exit -; COMMON-SAME: () #[[ATTR2]] { -; COMMON-NEXT: entry: -; COMMON-NEXT: br label [[WHILE_BODY:%.*]] -; COMMON: while.body: -; COMMON-NEXT: call void @cold0() -; COMMON-NEXT: br label [[WHILE_BODY]] +; FNATTRS: Function Attrs: cold noreturn +; FNATTRS-LABEL: define void @test_no_exit +; FNATTRS-SAME: () #[[ATTR3:[0-9]+]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: br label [[WHILE_BODY:%.*]] +; FNATTRS: while.body: +; FNATTRS-NEXT: call void @cold0() +; FNATTRS-NEXT: br label [[WHILE_BODY]] +; +; ATTRIBUTOR: Function Attrs: noreturn +; ATTRIBUTOR-LABEL: define void @test_no_exit +; ATTRIBUTOR-SAME: () #[[ATTR2]] { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY:%.*]] +; ATTRIBUTOR: while.body: +; ATTRIBUTOR-NEXT: call void @cold0() +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY]] ; entry: br label %while.body @@ -72,17 +81,29 @@ while.body: } define void @test_no_exit2() { -; COMMON: Function Attrs: noreturn -; COMMON-LABEL: define void @test_no_exit2 -; COMMON-SAME: () #[[ATTR2]] { -; COMMON-NEXT: entry: -; COMMON-NEXT: br label [[WHILE_BODY:%.*]] -; COMMON: while.body: -; COMMON-NEXT: call void @not_cold0() -; COMMON-NEXT: br label [[WHILE_BODY2:%.*]] -; COMMON: while.body2: -; COMMON-NEXT: call void @cold1() -; COMMON-NEXT: br label [[WHILE_BODY]] +; FNATTRS: Function Attrs: cold noreturn +; FNATTRS-LABEL: define void @test_no_exit2 +; FNATTRS-SAME: () #[[ATTR3]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: br label [[WHILE_BODY:%.*]] +; FNATTRS: while.body: +; FNATTRS-NEXT: call void @not_cold0() +; FNATTRS-NEXT: br label [[WHILE_BODY2:%.*]] +; FNATTRS: while.body2: +; FNATTRS-NEXT: call void @cold1() +; FNATTRS-NEXT: br label [[WHILE_BODY]] +; +; ATTRIBUTOR: Function Attrs: noreturn +; ATTRIBUTOR-LABEL: define void @test_no_exit2 +; ATTRIBUTOR-SAME: () #[[ATTR2]] { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY:%.*]] +; ATTRIBUTOR: while.body: +; ATTRIBUTOR-NEXT: call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY2:%.*]] +; ATTRIBUTOR: while.body2: +; ATTRIBUTOR-NEXT: call void @cold1() +; ATTRIBUTOR-NEXT: br label [[WHILE_BODY]] ; entry: br label %while.body @@ -97,18 +118,32 @@ while.body2: } define dso_local void @test_entry(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_entry -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: br label [[IF_END]] -; COMMON: if.end: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_entry +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0:[0-9]+]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: br label [[IF_END]] +; FNATTRS: if.end: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_entry +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END]] +; ATTRIBUTOR: if.end: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @cold0() @@ -125,12 +160,19 @@ if.end: } define dso_local void @test_hot_fail(i32 noundef %x) hot { -; COMMON: Function Attrs: hot -; COMMON-LABEL: define dso_local void @test_hot_fail -; COMMON-SAME: (i32 noundef [[X:%.*]]) #[[ATTR3:[0-9]+]] { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: hot +; FNATTRS-LABEL: define dso_local void @test_hot_fail +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR4:[0-9]+]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR: Function Attrs: hot +; ATTRIBUTOR-LABEL: define dso_local void @test_hot_fail +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) #[[ATTR3:[0-9]+]] { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @cold0() @@ -138,19 +180,34 @@ entry: } define dso_local void @test_br2(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_br2 -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END:%.*]] -; COMMON: if.else: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END]] -; COMMON: if.end: -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_br2 +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END]] +; FNATTRS: if.end: +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_br2 +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END]] +; ATTRIBUTOR: if.end: +; ATTRIBUTOR-NEXT: ret void ; entry: %tobool.not = icmp eq i32 %x, 0 @@ -169,21 +226,38 @@ if.end: } define dso_local void @test_exit(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_exit -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: br label [[IF_END:%.*]] -; COMMON: if.else: -; COMMON-NEXT: tail call void @not_cold2() -; COMMON-NEXT: br label [[IF_END]] -; COMMON: if.end: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_exit +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: br label [[IF_END:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: tail call void @not_cold2() +; FNATTRS-NEXT: br label [[IF_END]] +; FNATTRS: if.end: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_exit +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: tail call void @not_cold2() +; ATTRIBUTOR-NEXT: br label [[IF_END]] +; ATTRIBUTOR: if.end: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @not_cold0() @@ -204,54 +278,104 @@ if.end: } define dso_local void @test_complex(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_complex -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: [[CALL:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL]], 0 -; COMMON-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] -; COMMON: if.then2: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END12:%.*]] -; COMMON: if.else: -; COMMON-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 -; COMMON-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] -; COMMON: if.then5: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: if.else6: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ -; COMMON-NEXT: i32 0, label [[SW_BB:%.*]] -; COMMON-NEXT: i32 1, label [[SW_BB8:%.*]] -; COMMON-NEXT: i32 2, label [[SW_BB9:%.*]] -; COMMON-NEXT: ] -; COMMON: sw.bb: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: br label [[CALL_COLD:%.*]] -; COMMON: sw.bb8: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.bb9: -; COMMON-NEXT: tail call void @not_cold2() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.default: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: call_cold: -; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0:[0-9]+]] -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: if.else11: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END12]] -; COMMON: if.end12: -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_complex +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: [[CALL:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; FNATTRS: if.then2: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END12:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; FNATTRS: if.then5: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: if.else6: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; FNATTRS-NEXT: i32 0, label [[SW_BB:%.*]] +; FNATTRS-NEXT: i32 1, label [[SW_BB8:%.*]] +; FNATTRS-NEXT: i32 2, label [[SW_BB9:%.*]] +; FNATTRS-NEXT: ] +; FNATTRS: sw.bb: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: br label [[CALL_COLD:%.*]] +; FNATTRS: sw.bb8: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.bb9: +; FNATTRS-NEXT: tail call void @not_cold2() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.default: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: call_cold: +; FNATTRS-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: if.else11: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END12]] +; FNATTRS: if.end12: +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_complex +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: [[CALL:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; ATTRIBUTOR: if.then2: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END12:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; ATTRIBUTOR: if.then5: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: if.else6: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; ATTRIBUTOR-NEXT: i32 0, label [[SW_BB:%.*]] +; ATTRIBUTOR-NEXT: i32 1, label [[SW_BB8:%.*]] +; ATTRIBUTOR-NEXT: i32 2, label [[SW_BB9:%.*]] +; ATTRIBUTOR-NEXT: ] +; ATTRIBUTOR: sw.bb: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD:%.*]] +; ATTRIBUTOR: sw.bb8: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.bb9: +; ATTRIBUTOR-NEXT: tail call void @not_cold2() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.default: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: call_cold: +; ATTRIBUTOR-NEXT: tail call void @cold_at_cb() #[[ATTR0:[0-9]+]] +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: if.else11: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END12]] +; ATTRIBUTOR: if.end12: +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @not_cold0() @@ -314,63 +438,122 @@ if.end12: } define dso_local void @test_complex2(i32 noundef %x) { -; COMMON-LABEL: define dso_local void @test_complex2 -; COMMON-SAME: (i32 noundef [[X:%.*]]) { -; COMMON-NEXT: entry: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 -; COMMON-NEXT: [[CALL12:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] -; COMMON: if.then: -; COMMON-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL12]], 0 -; COMMON-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] -; COMMON: if.then2: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END16:%.*]] -; COMMON: if.else: -; COMMON-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 -; COMMON-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] -; COMMON: if.then5: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: if.else6: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() -; COMMON-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ -; COMMON-NEXT: i32 0, label [[SW_BB:%.*]] -; COMMON-NEXT: i32 1, label [[SW_BB8:%.*]] -; COMMON-NEXT: i32 2, label [[SW_BB9:%.*]] -; COMMON-NEXT: ] -; COMMON: sw.bb: -; COMMON-NEXT: tail call void @not_cold0() -; COMMON-NEXT: br label [[CALL_COLD:%.*]] -; COMMON: sw.bb8: -; COMMON-NEXT: tail call void @not_cold1() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.bb9: -; COMMON-NEXT: tail call void @not_cold2() -; COMMON-NEXT: br label [[CALL_COLD]] -; COMMON: sw.default: -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: call_cold: -; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0]] -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: if.else11: -; COMMON-NEXT: [[CMP:%.*]] = icmp slt i32 [[CALL12]], 1 -; COMMON-NEXT: br i1 [[CMP]], label [[IF_END14:%.*]], label [[FOR_BODY:%.*]] -; COMMON: if.end14: -; COMMON-NEXT: tail call void @cold1() -; COMMON-NEXT: br label [[IF_END16]] -; COMMON: for.body: -; COMMON-NEXT: [[I_021:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[IF_ELSE11]] ] -; COMMON-NEXT: tail call void @cold0() -; COMMON-NEXT: [[INC]] = add nuw nsw i32 [[I_021]], 1 -; COMMON-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[CALL12]] -; COMMON-NEXT: br i1 [[EXITCOND_NOT]], label [[IF_END16]], label [[FOR_BODY]] -; COMMON: if.end16: -; COMMON-NEXT: ret void +; FNATTRS: Function Attrs: cold +; FNATTRS-LABEL: define dso_local void @test_complex2 +; FNATTRS-SAME: (i32 noundef [[X:%.*]]) #[[ATTR0]] { +; FNATTRS-NEXT: entry: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; FNATTRS-NEXT: [[CALL12:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; FNATTRS: if.then: +; FNATTRS-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL12]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; FNATTRS: if.then2: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END16:%.*]] +; FNATTRS: if.else: +; FNATTRS-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; FNATTRS-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; FNATTRS: if.then5: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: if.else6: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; FNATTRS-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; FNATTRS-NEXT: i32 0, label [[SW_BB:%.*]] +; FNATTRS-NEXT: i32 1, label [[SW_BB8:%.*]] +; FNATTRS-NEXT: i32 2, label [[SW_BB9:%.*]] +; FNATTRS-NEXT: ] +; FNATTRS: sw.bb: +; FNATTRS-NEXT: tail call void @not_cold0() +; FNATTRS-NEXT: br label [[CALL_COLD:%.*]] +; FNATTRS: sw.bb8: +; FNATTRS-NEXT: tail call void @not_cold1() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.bb9: +; FNATTRS-NEXT: tail call void @not_cold2() +; FNATTRS-NEXT: br label [[CALL_COLD]] +; FNATTRS: sw.default: +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: call_cold: +; FNATTRS-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: if.else11: +; FNATTRS-NEXT: [[CMP:%.*]] = icmp slt i32 [[CALL12]], 1 +; FNATTRS-NEXT: br i1 [[CMP]], label [[IF_END14:%.*]], label [[FOR_BODY:%.*]] +; FNATTRS: if.end14: +; FNATTRS-NEXT: tail call void @cold1() +; FNATTRS-NEXT: br label [[IF_END16]] +; FNATTRS: for.body: +; FNATTRS-NEXT: [[I_021:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[IF_ELSE11]] ] +; FNATTRS-NEXT: tail call void @cold0() +; FNATTRS-NEXT: [[INC]] = add nuw nsw i32 [[I_021]], 1 +; FNATTRS-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[CALL12]] +; FNATTRS-NEXT: br i1 [[EXITCOND_NOT]], label [[IF_END16]], label [[FOR_BODY]] +; FNATTRS: if.end16: +; FNATTRS-NEXT: ret void +; +; ATTRIBUTOR-LABEL: define dso_local void @test_complex2 +; ATTRIBUTOR-SAME: (i32 noundef [[X:%.*]]) { +; ATTRIBUTOR-NEXT: entry: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[X]], 0 +; ATTRIBUTOR-NEXT: [[CALL12:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_ELSE11:%.*]], label [[IF_THEN:%.*]] +; ATTRIBUTOR: if.then: +; ATTRIBUTOR-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[CALL12]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL1_NOT]], label [[IF_ELSE:%.*]], label [[IF_THEN2:%.*]] +; ATTRIBUTOR: if.then2: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END16:%.*]] +; ATTRIBUTOR: if.else: +; ATTRIBUTOR-NEXT: [[CALL3:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: [[TOBOOL4_NOT:%.*]] = icmp eq i32 [[CALL3]], 0 +; ATTRIBUTOR-NEXT: br i1 [[TOBOOL4_NOT]], label [[IF_ELSE6:%.*]], label [[IF_THEN5:%.*]] +; ATTRIBUTOR: if.then5: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: if.else6: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: [[CALL7:%.*]] = tail call i32 @get_val() +; ATTRIBUTOR-NEXT: switch i32 [[CALL7]], label [[SW_DEFAULT:%.*]] [ +; ATTRIBUTOR-NEXT: i32 0, label [[SW_BB:%.*]] +; ATTRIBUTOR-NEXT: i32 1, label [[SW_BB8:%.*]] +; ATTRIBUTOR-NEXT: i32 2, label [[SW_BB9:%.*]] +; ATTRIBUTOR-NEXT: ] +; ATTRIBUTOR: sw.bb: +; ATTRIBUTOR-NEXT: tail call void @not_cold0() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD:%.*]] +; ATTRIBUTOR: sw.bb8: +; ATTRIBUTOR-NEXT: tail call void @not_cold1() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.bb9: +; ATTRIBUTOR-NEXT: tail call void @not_cold2() +; ATTRIBUTOR-NEXT: br label [[CALL_COLD]] +; ATTRIBUTOR: sw.default: +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: call_cold: +; ATTRIBUTOR-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: if.else11: +; ATTRIBUTOR-NEXT: [[CMP:%.*]] = icmp slt i32 [[CALL12]], 1 +; ATTRIBUTOR-NEXT: br i1 [[CMP]], label [[IF_END14:%.*]], label [[FOR_BODY:%.*]] +; ATTRIBUTOR: if.end14: +; ATTRIBUTOR-NEXT: tail call void @cold1() +; ATTRIBUTOR-NEXT: br label [[IF_END16]] +; ATTRIBUTOR: for.body: +; ATTRIBUTOR-NEXT: [[I_021:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[IF_ELSE11]] ] +; ATTRIBUTOR-NEXT: tail call void @cold0() +; ATTRIBUTOR-NEXT: [[INC]] = add nuw nsw i32 [[I_021]], 1 +; ATTRIBUTOR-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], [[CALL12]] +; ATTRIBUTOR-NEXT: br i1 [[EXITCOND_NOT]], label [[IF_END16]], label [[FOR_BODY]] +; ATTRIBUTOR: if.end16: +; ATTRIBUTOR-NEXT: ret void ; entry: tail call void @not_cold0() @@ -485,7 +668,7 @@ define dso_local void @test_complex_fail(i32 noundef %x) { ; COMMON-NEXT: tail call void @cold0() ; COMMON-NEXT: br label [[IF_END12]] ; COMMON: call_cold: -; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0]] +; COMMON-NEXT: tail call void @cold_at_cb() #[[ATTR0:[0-9]+]] ; COMMON-NEXT: br label [[IF_END12]] ; COMMON: if.else11: ; COMMON-NEXT: tail call void @cold0() @@ -684,11 +867,14 @@ if.end16: } ;. -; COMMON: attributes #[[ATTR0]] = { cold } -; COMMON: attributes #[[ATTR1]] = { nofree norecurse noreturn nosync nounwind memory(none) } -; COMMON: attributes #[[ATTR2]] = { noreturn } -; COMMON: attributes #[[ATTR3]] = { hot } +; FNATTRS: attributes #[[ATTR0]] = { cold } +; FNATTRS: attributes #[[ATTR1]] = { nofree norecurse noreturn nosync nounwind memory(none) } +; FNATTRS: attributes #[[ATTR2]] = { noreturn } +; FNATTRS: attributes #[[ATTR3]] = { cold noreturn } +; FNATTRS: attributes #[[ATTR4]] = { hot } +;. +; ATTRIBUTOR: attributes #[[ATTR0]] = { cold } +; ATTRIBUTOR: attributes #[[ATTR1]] = { nofree norecurse noreturn nosync nounwind memory(none) } +; ATTRIBUTOR: attributes #[[ATTR2]] = { noreturn } +; ATTRIBUTOR: attributes #[[ATTR3]] = { hot } ;. -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; ATTRIBUTOR: {{.*}} -; FNATTRS: {{.*}}