2003-06-22 20:10:28 +00:00
|
|
|
//===- TailDuplication.cpp - Simplify CFG through tail duplication --------===//
|
2005-04-21 23:48:37 +00:00
|
|
|
//
|
2003-10-20 19:43:21 +00:00
|
|
|
// The LLVM Compiler Infrastructure
|
|
|
|
//
|
2007-12-29 20:36:04 +00:00
|
|
|
// This file is distributed under the University of Illinois Open Source
|
|
|
|
// License. See LICENSE.TXT for details.
|
2005-04-21 23:48:37 +00:00
|
|
|
//
|
2003-10-20 19:43:21 +00:00
|
|
|
//===----------------------------------------------------------------------===//
|
2003-06-22 20:10:28 +00:00
|
|
|
//
|
|
|
|
// This pass performs a limited form of tail duplication, intended to simplify
|
|
|
|
// CFGs by removing some unconditional branches. This pass is necessary to
|
|
|
|
// straighten out loops created by the C front-end, but also is capable of
|
|
|
|
// making other code nicer. After this pass is run, the CFG simplify pass
|
|
|
|
// should be run to clean up the mess.
|
|
|
|
//
|
|
|
|
// This pass could be enhanced in the future to use profile information to be
|
|
|
|
// more aggressive.
|
|
|
|
//
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
|
2006-09-27 04:58:23 +00:00
|
|
|
#define DEBUG_TYPE "tailduplicate"
|
2003-06-22 20:10:28 +00:00
|
|
|
#include "llvm/Transforms/Scalar.h"
|
2003-08-31 21:17:44 +00:00
|
|
|
#include "llvm/Constant.h"
|
2003-06-22 20:10:28 +00:00
|
|
|
#include "llvm/Function.h"
|
2004-07-29 17:05:13 +00:00
|
|
|
#include "llvm/Instructions.h"
|
2004-11-22 17:23:57 +00:00
|
|
|
#include "llvm/IntrinsicInst.h"
|
2003-06-22 20:10:28 +00:00
|
|
|
#include "llvm/Pass.h"
|
|
|
|
#include "llvm/Type.h"
|
|
|
|
#include "llvm/Support/CFG.h"
|
|
|
|
#include "llvm/Transforms/Utils/Local.h"
|
2004-09-01 22:55:40 +00:00
|
|
|
#include "llvm/Support/CommandLine.h"
|
2007-02-05 23:32:05 +00:00
|
|
|
#include "llvm/Support/Compiler.h"
|
2004-09-01 22:55:40 +00:00
|
|
|
#include "llvm/Support/Debug.h"
|
|
|
|
#include "llvm/ADT/Statistic.h"
|
2008-05-13 20:06:43 +00:00
|
|
|
#include "llvm/ADT/SmallPtrSet.h"
|
2008-03-21 23:51:57 +00:00
|
|
|
#include <map>
|
2004-01-09 06:02:20 +00:00
|
|
|
using namespace llvm;
|
2003-11-11 22:41:34 +00:00
|
|
|
|
2006-12-19 21:40:18 +00:00
|
|
|
STATISTIC(NumEliminated, "Number of unconditional branches eliminated");
|
|
|
|
|
2008-05-13 00:00:25 +00:00
|
|
|
static cl::opt<unsigned>
|
|
|
|
Threshold("taildup-threshold", cl::desc("Max block size to tail duplicate"),
|
|
|
|
cl::init(6), cl::Hidden);
|
|
|
|
|
2003-06-22 20:10:28 +00:00
|
|
|
namespace {
|
2007-02-05 23:32:05 +00:00
|
|
|
class VISIBILITY_HIDDEN TailDup : public FunctionPass {
|
2003-06-22 20:10:28 +00:00
|
|
|
bool runOnFunction(Function &F);
|
2007-05-01 21:15:47 +00:00
|
|
|
public:
|
2007-05-06 13:37:16 +00:00
|
|
|
static char ID; // Pass identification, replacement for typeid
|
2007-05-01 21:15:47 +00:00
|
|
|
TailDup() : FunctionPass((intptr_t)&ID) {}
|
|
|
|
|
2003-06-22 20:10:28 +00:00
|
|
|
private:
|
|
|
|
inline bool shouldEliminateUnconditionalBranch(TerminatorInst *TI);
|
|
|
|
inline void eliminateUnconditionalBranch(BranchInst *BI);
|
2008-05-13 20:06:43 +00:00
|
|
|
SmallPtrSet<BasicBlock*, 4> CycleDetector;
|
2003-06-22 20:10:28 +00:00
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2008-05-13 00:00:25 +00:00
|
|
|
char TailDup::ID = 0;
|
|
|
|
static RegisterPass<TailDup> X("tailduplicate", "Tail Duplication");
|
|
|
|
|
2003-11-11 22:41:34 +00:00
|
|
|
// Public interface to the Tail Duplication pass
|
2004-09-20 04:43:15 +00:00
|
|
|
FunctionPass *llvm::createTailDuplicationPass() { return new TailDup(); }
|
2003-06-22 20:10:28 +00:00
|
|
|
|
|
|
|
/// runOnFunction - Top level algorithm - Loop over each unconditional branch in
|
2008-05-13 20:06:43 +00:00
|
|
|
/// the function, eliminating it if it looks attractive enough. CycleDetector
|
|
|
|
/// prevents infinite loops by checking that we aren't redirecting a branch to
|
|
|
|
/// a place it already pointed to earlier; see PR 2323.
|
2003-06-22 20:10:28 +00:00
|
|
|
bool TailDup::runOnFunction(Function &F) {
|
|
|
|
bool Changed = false;
|
2008-05-13 20:06:43 +00:00
|
|
|
CycleDetector.clear();
|
|
|
|
for (Function::iterator I = F.begin(), E = F.end(); I != E; ) {
|
Okay, so there is no reasonable way for tail duplication to update SSA form,
as it is making effectively arbitrary modifications to the CFG and we don't
have a domset/domfrontier implementations that can handle the dynamic updates.
Instead of having a bunch of code that doesn't actually work in practice,
just demote any potentially tricky values to the stack (causing the problem
to go away entirely). Later invocations of mem2reg will rebuild SSA for us.
This fixes all of the major performance regressions with tail duplication
from LLVM 1.1. For example, this loop:
---
int popcount(int x) {
int result = 0;
while (x != 0) {
result = result + (x & 0x1);
x = x >> 1;
}
return result;
}
---
Used to be compiled into:
int %popcount(int %X) {
entry:
br label %loopentry
loopentry: ; preds = %entry, %no_exit
%x.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=3]
%result.1.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=2]
%tmp.1 = seteq int %x.0, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
no_exit: ; preds = %loopentry
%tmp.4 = and int %x.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0 ; <int> [#uses=1]
%tmp.9 = shr int %x.0, ubyte 1 ; <int> [#uses=1]
br label %loopentry
loopexit: ; preds = %loopentry
ret int %result.1.0
}
And is now compiled into:
int %popcount(int %X) {
entry:
br label %no_exit
no_exit: ; preds = %entry, %no_exit
%x.0.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=2]
%result.1.0.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=1]
%tmp.4 = and int %x.0.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0.0 ; <int> [#uses=2]
%tmp.9 = shr int %x.0.0, ubyte 1 ; <int> [#uses=2]
%tmp.1 = seteq int %tmp.9, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
loopexit: ; preds = %no_exit
ret int %tmp.6
}
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@12457 91177308-0d34-0410-b5e6-96231b3b80d8
2004-03-16 23:29:09 +00:00
|
|
|
if (shouldEliminateUnconditionalBranch(I->getTerminator())) {
|
2003-06-22 20:10:28 +00:00
|
|
|
eliminateUnconditionalBranch(cast<BranchInst>(I->getTerminator()));
|
|
|
|
Changed = true;
|
|
|
|
} else {
|
|
|
|
++I;
|
2008-05-13 20:06:43 +00:00
|
|
|
CycleDetector.clear();
|
2003-06-22 20:10:28 +00:00
|
|
|
}
|
2008-05-13 20:06:43 +00:00
|
|
|
}
|
2003-06-22 20:10:28 +00:00
|
|
|
return Changed;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// shouldEliminateUnconditionalBranch - Return true if this branch looks
|
|
|
|
/// attractive to eliminate. We eliminate the branch if the destination basic
|
|
|
|
/// block has <= 5 instructions in it, not counting PHI nodes. In practice,
|
|
|
|
/// since one of these is a terminator instruction, this means that we will add
|
|
|
|
/// up to 4 instructions to the new block.
|
|
|
|
///
|
|
|
|
/// We don't count PHI nodes in the count since they will be removed when the
|
|
|
|
/// contents of the block are copied over.
|
|
|
|
///
|
|
|
|
bool TailDup::shouldEliminateUnconditionalBranch(TerminatorInst *TI) {
|
|
|
|
BranchInst *BI = dyn_cast<BranchInst>(TI);
|
|
|
|
if (!BI || !BI->isUnconditional()) return false; // Not an uncond branch!
|
|
|
|
|
|
|
|
BasicBlock *Dest = BI->getSuccessor(0);
|
|
|
|
if (Dest == BI->getParent()) return false; // Do not loop infinitely!
|
|
|
|
|
2003-07-23 03:32:41 +00:00
|
|
|
// Do not inline a block if we will just get another branch to the same block!
|
2004-03-16 19:45:22 +00:00
|
|
|
TerminatorInst *DTI = Dest->getTerminator();
|
|
|
|
if (BranchInst *DBI = dyn_cast<BranchInst>(DTI))
|
2003-07-23 03:32:41 +00:00
|
|
|
if (DBI->isUnconditional() && DBI->getSuccessor(0) == Dest)
|
|
|
|
return false; // Do not loop infinitely!
|
|
|
|
|
Okay, so there is no reasonable way for tail duplication to update SSA form,
as it is making effectively arbitrary modifications to the CFG and we don't
have a domset/domfrontier implementations that can handle the dynamic updates.
Instead of having a bunch of code that doesn't actually work in practice,
just demote any potentially tricky values to the stack (causing the problem
to go away entirely). Later invocations of mem2reg will rebuild SSA for us.
This fixes all of the major performance regressions with tail duplication
from LLVM 1.1. For example, this loop:
---
int popcount(int x) {
int result = 0;
while (x != 0) {
result = result + (x & 0x1);
x = x >> 1;
}
return result;
}
---
Used to be compiled into:
int %popcount(int %X) {
entry:
br label %loopentry
loopentry: ; preds = %entry, %no_exit
%x.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=3]
%result.1.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=2]
%tmp.1 = seteq int %x.0, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
no_exit: ; preds = %loopentry
%tmp.4 = and int %x.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0 ; <int> [#uses=1]
%tmp.9 = shr int %x.0, ubyte 1 ; <int> [#uses=1]
br label %loopentry
loopexit: ; preds = %loopentry
ret int %result.1.0
}
And is now compiled into:
int %popcount(int %X) {
entry:
br label %no_exit
no_exit: ; preds = %entry, %no_exit
%x.0.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=2]
%result.1.0.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=1]
%tmp.4 = and int %x.0.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0.0 ; <int> [#uses=2]
%tmp.9 = shr int %x.0.0, ubyte 1 ; <int> [#uses=2]
%tmp.1 = seteq int %tmp.9, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
loopexit: ; preds = %no_exit
ret int %tmp.6
}
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@12457 91177308-0d34-0410-b5e6-96231b3b80d8
2004-03-16 23:29:09 +00:00
|
|
|
// FIXME: DemoteRegToStack cannot yet demote invoke instructions to the stack,
|
|
|
|
// because doing so would require breaking critical edges. This should be
|
|
|
|
// fixed eventually.
|
|
|
|
if (!DTI->use_empty())
|
|
|
|
return false;
|
|
|
|
|
2003-06-22 20:10:28 +00:00
|
|
|
// Do not bother working on dead blocks...
|
|
|
|
pred_iterator PI = pred_begin(Dest), PE = pred_end(Dest);
|
|
|
|
if (PI == PE && Dest != Dest->getParent()->begin())
|
|
|
|
return false; // It's just a dead block, ignore it...
|
|
|
|
|
|
|
|
// Also, do not bother with blocks with only a single predecessor: simplify
|
|
|
|
// CFG will fold these two blocks together!
|
|
|
|
++PI;
|
|
|
|
if (PI == PE) return false; // Exactly one predecessor!
|
|
|
|
|
|
|
|
BasicBlock::iterator I = Dest->begin();
|
|
|
|
while (isa<PHINode>(*I)) ++I;
|
|
|
|
|
2004-11-22 17:23:57 +00:00
|
|
|
for (unsigned Size = 0; I != Dest->end(); ++I) {
|
|
|
|
if (Size == Threshold) return false; // The block is too large.
|
2007-11-04 06:37:55 +00:00
|
|
|
|
|
|
|
// Don't tail duplicate call instructions. They are very large compared to
|
|
|
|
// other instructions.
|
|
|
|
if (isa<CallInst>(I) || isa<InvokeInst>(I)) return false;
|
|
|
|
|
2004-11-22 17:23:57 +00:00
|
|
|
// Only count instructions that are not debugger intrinsics.
|
|
|
|
if (!isa<DbgInfoIntrinsic>(I)) ++Size;
|
|
|
|
}
|
2004-03-16 19:45:22 +00:00
|
|
|
|
|
|
|
// Do not tail duplicate a block that has thousands of successors into a block
|
|
|
|
// with a single successor if the block has many other predecessors. This can
|
|
|
|
// cause an N^2 explosion in CFG edges (and PHI node entries), as seen in
|
|
|
|
// cases that have a large number of indirect gotos.
|
2004-11-01 07:05:07 +00:00
|
|
|
unsigned NumSuccs = DTI->getNumSuccessors();
|
|
|
|
if (NumSuccs > 8) {
|
|
|
|
unsigned TooMany = 128;
|
|
|
|
if (NumSuccs >= TooMany) return false;
|
|
|
|
TooMany = TooMany/NumSuccs;
|
|
|
|
for (; PI != PE; ++PI)
|
|
|
|
if (TooMany-- == 0) return false;
|
|
|
|
}
|
Throttle back tail duplication to avoid creating really ugly sequences of code.
For Transforms/TailDup/if-tail-dup.ll, f.e., it produces:
_foo:
movl 8(%esp), %eax
movl 4(%esp), %ecx
testl $1, %ecx
je LBB1_2 #cond_next
LBB1_1: #cond_true
movl $1, (%eax)
LBB1_2: #cond_next
testl $2, %ecx
je LBB1_4 #cond_next10
LBB1_3: #cond_true6
movl $1, 4(%eax)
LBB1_4: #cond_next10
testl $4, %ecx
je LBB1_6 #cond_next18
LBB1_5: #cond_true14
movl $1, 8(%eax)
LBB1_6: #cond_next18
testl $8, %ecx
je LBB1_8 #return
LBB1_7: #cond_true22
movl $1, 12(%eax)
ret
LBB1_8: #return
ret
instead of:
_foo:
movl 4(%esp), %eax
testl $2, %eax
sete %cl
movl 8(%esp), %edx
testl $1, %eax
je LBB1_2 #cond_next
LBB1_1: #cond_true
movl $1, (%edx)
testb %cl, %cl
jne LBB1_4 #cond_next10
jmp LBB1_3 #cond_true6
LBB1_2: #cond_next
testb %cl, %cl
jne LBB1_4 #cond_next10
LBB1_3: #cond_true6
movl $1, 4(%edx)
testl $4, %eax
je LBB1_6 #cond_next18
jmp LBB1_5 #cond_true14
LBB1_4: #cond_next10
testl $4, %eax
je LBB1_6 #cond_next18
LBB1_5: #cond_true14
movl $1, 8(%edx)
testl $8, %eax
je LBB1_8 #return
jmp LBB1_7 #cond_true22
LBB1_6: #cond_next18
testl $8, %eax
je LBB1_8 #return
LBB1_7: #cond_true22
movl $1, 12(%edx)
ret
LBB1_8: #return
ret
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@30158 91177308-0d34-0410-b5e6-96231b3b80d8
2006-09-07 21:30:15 +00:00
|
|
|
|
2008-05-13 20:06:43 +00:00
|
|
|
// If this unconditional branch is a fall-through, be careful about
|
Throttle back tail duplication to avoid creating really ugly sequences of code.
For Transforms/TailDup/if-tail-dup.ll, f.e., it produces:
_foo:
movl 8(%esp), %eax
movl 4(%esp), %ecx
testl $1, %ecx
je LBB1_2 #cond_next
LBB1_1: #cond_true
movl $1, (%eax)
LBB1_2: #cond_next
testl $2, %ecx
je LBB1_4 #cond_next10
LBB1_3: #cond_true6
movl $1, 4(%eax)
LBB1_4: #cond_next10
testl $4, %ecx
je LBB1_6 #cond_next18
LBB1_5: #cond_true14
movl $1, 8(%eax)
LBB1_6: #cond_next18
testl $8, %ecx
je LBB1_8 #return
LBB1_7: #cond_true22
movl $1, 12(%eax)
ret
LBB1_8: #return
ret
instead of:
_foo:
movl 4(%esp), %eax
testl $2, %eax
sete %cl
movl 8(%esp), %edx
testl $1, %eax
je LBB1_2 #cond_next
LBB1_1: #cond_true
movl $1, (%edx)
testb %cl, %cl
jne LBB1_4 #cond_next10
jmp LBB1_3 #cond_true6
LBB1_2: #cond_next
testb %cl, %cl
jne LBB1_4 #cond_next10
LBB1_3: #cond_true6
movl $1, 4(%edx)
testl $4, %eax
je LBB1_6 #cond_next18
jmp LBB1_5 #cond_true14
LBB1_4: #cond_next10
testl $4, %eax
je LBB1_6 #cond_next18
LBB1_5: #cond_true14
movl $1, 8(%edx)
testl $8, %eax
je LBB1_8 #return
jmp LBB1_7 #cond_true22
LBB1_6: #cond_next18
testl $8, %eax
je LBB1_8 #return
LBB1_7: #cond_true22
movl $1, 12(%edx)
ret
LBB1_8: #return
ret
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@30158 91177308-0d34-0410-b5e6-96231b3b80d8
2006-09-07 21:30:15 +00:00
|
|
|
// tail duplicating it. In particular, we don't want to taildup it if the
|
|
|
|
// original block will still be there after taildup is completed: doing so
|
|
|
|
// would eliminate the fall-through, requiring unconditional branches.
|
|
|
|
Function::iterator DestI = Dest;
|
|
|
|
if (&*--DestI == BI->getParent()) {
|
|
|
|
// The uncond branch is a fall-through. Tail duplication of the block is
|
|
|
|
// will eliminate the fall-through-ness and end up cloning the terminator
|
|
|
|
// at the end of the Dest block. Since the original Dest block will
|
|
|
|
// continue to exist, this means that one or the other will not be able to
|
|
|
|
// fall through. One typical example that this helps with is code like:
|
|
|
|
// if (a)
|
|
|
|
// foo();
|
|
|
|
// if (b)
|
|
|
|
// foo();
|
|
|
|
// Cloning the 'if b' block into the end of the first foo block is messy.
|
2006-09-10 18:17:58 +00:00
|
|
|
|
|
|
|
// The messy case is when the fall-through block falls through to other
|
|
|
|
// blocks. This is what we would be preventing if we cloned the block.
|
|
|
|
DestI = Dest;
|
|
|
|
if (++DestI != Dest->getParent()->end()) {
|
|
|
|
BasicBlock *DestSucc = DestI;
|
|
|
|
// If any of Dest's successors are fall-throughs, don't do this xform.
|
|
|
|
for (succ_iterator SI = succ_begin(Dest), SE = succ_end(Dest);
|
|
|
|
SI != SE; ++SI)
|
|
|
|
if (*SI == DestSucc)
|
|
|
|
return false;
|
|
|
|
}
|
Throttle back tail duplication to avoid creating really ugly sequences of code.
For Transforms/TailDup/if-tail-dup.ll, f.e., it produces:
_foo:
movl 8(%esp), %eax
movl 4(%esp), %ecx
testl $1, %ecx
je LBB1_2 #cond_next
LBB1_1: #cond_true
movl $1, (%eax)
LBB1_2: #cond_next
testl $2, %ecx
je LBB1_4 #cond_next10
LBB1_3: #cond_true6
movl $1, 4(%eax)
LBB1_4: #cond_next10
testl $4, %ecx
je LBB1_6 #cond_next18
LBB1_5: #cond_true14
movl $1, 8(%eax)
LBB1_6: #cond_next18
testl $8, %ecx
je LBB1_8 #return
LBB1_7: #cond_true22
movl $1, 12(%eax)
ret
LBB1_8: #return
ret
instead of:
_foo:
movl 4(%esp), %eax
testl $2, %eax
sete %cl
movl 8(%esp), %edx
testl $1, %eax
je LBB1_2 #cond_next
LBB1_1: #cond_true
movl $1, (%edx)
testb %cl, %cl
jne LBB1_4 #cond_next10
jmp LBB1_3 #cond_true6
LBB1_2: #cond_next
testb %cl, %cl
jne LBB1_4 #cond_next10
LBB1_3: #cond_true6
movl $1, 4(%edx)
testl $4, %eax
je LBB1_6 #cond_next18
jmp LBB1_5 #cond_true14
LBB1_4: #cond_next10
testl $4, %eax
je LBB1_6 #cond_next18
LBB1_5: #cond_true14
movl $1, 8(%edx)
testl $8, %eax
je LBB1_8 #return
jmp LBB1_7 #cond_true22
LBB1_6: #cond_next18
testl $8, %eax
je LBB1_8 #return
LBB1_7: #cond_true22
movl $1, 12(%edx)
ret
LBB1_8: #return
ret
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@30158 91177308-0d34-0410-b5e6-96231b3b80d8
2006-09-07 21:30:15 +00:00
|
|
|
}
|
2004-03-16 19:45:22 +00:00
|
|
|
|
2008-05-13 20:06:43 +00:00
|
|
|
// Finally, check that we haven't redirected to this target block earlier;
|
|
|
|
// there are cases where we loop forever if we don't check this (PR 2323).
|
|
|
|
if (!CycleDetector.insert(Dest))
|
|
|
|
return false;
|
|
|
|
|
2005-04-21 23:48:37 +00:00
|
|
|
return true;
|
2003-06-22 20:10:28 +00:00
|
|
|
}
|
2004-02-01 06:32:28 +00:00
|
|
|
|
2004-10-06 03:27:37 +00:00
|
|
|
/// FindObviousSharedDomOf - We know there is a branch from SrcBlock to
|
|
|
|
/// DestBlock, and that SrcBlock is not the only predecessor of DstBlock. If we
|
|
|
|
/// can find a predecessor of SrcBlock that is a dominator of both SrcBlock and
|
|
|
|
/// DstBlock, return it.
|
|
|
|
static BasicBlock *FindObviousSharedDomOf(BasicBlock *SrcBlock,
|
|
|
|
BasicBlock *DstBlock) {
|
|
|
|
// SrcBlock must have a single predecessor.
|
|
|
|
pred_iterator PI = pred_begin(SrcBlock), PE = pred_end(SrcBlock);
|
|
|
|
if (PI == PE || ++PI != PE) return 0;
|
|
|
|
|
|
|
|
BasicBlock *SrcPred = *pred_begin(SrcBlock);
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2004-10-06 03:27:37 +00:00
|
|
|
// Look at the predecessors of DstBlock. One of them will be SrcBlock. If
|
|
|
|
// there is only one other pred, get it, otherwise we can't handle it.
|
|
|
|
PI = pred_begin(DstBlock); PE = pred_end(DstBlock);
|
|
|
|
BasicBlock *DstOtherPred = 0;
|
|
|
|
if (*PI == SrcBlock) {
|
|
|
|
if (++PI == PE) return 0;
|
|
|
|
DstOtherPred = *PI;
|
|
|
|
if (++PI != PE) return 0;
|
|
|
|
} else {
|
|
|
|
DstOtherPred = *PI;
|
|
|
|
if (++PI == PE || *PI != SrcBlock || ++PI != PE) return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
// We can handle two situations here: "if then" and "if then else" blocks. An
|
|
|
|
// 'if then' situation is just where DstOtherPred == SrcPred.
|
|
|
|
if (DstOtherPred == SrcPred)
|
|
|
|
return SrcPred;
|
|
|
|
|
|
|
|
// Check to see if we have an "if then else" situation, which means that
|
|
|
|
// DstOtherPred will have a single predecessor and it will be SrcPred.
|
|
|
|
PI = pred_begin(DstOtherPred); PE = pred_end(DstOtherPred);
|
|
|
|
if (PI != PE && *PI == SrcPred) {
|
|
|
|
if (++PI != PE) return 0; // Not a single pred.
|
|
|
|
return SrcPred; // Otherwise, it's an "if then" situation. Return the if.
|
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise, this is something we can't handle.
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2003-06-22 20:10:28 +00:00
|
|
|
|
|
|
|
/// eliminateUnconditionalBranch - Clone the instructions from the destination
|
|
|
|
/// block into the source block, eliminating the specified unconditional branch.
|
|
|
|
/// If the destination block defines values used by successors of the dest
|
|
|
|
/// block, we may need to insert PHI nodes.
|
|
|
|
///
|
|
|
|
void TailDup::eliminateUnconditionalBranch(BranchInst *Branch) {
|
|
|
|
BasicBlock *SourceBlock = Branch->getParent();
|
|
|
|
BasicBlock *DestBlock = Branch->getSuccessor(0);
|
|
|
|
assert(SourceBlock != DestBlock && "Our predicate is broken!");
|
|
|
|
|
2006-11-26 09:46:52 +00:00
|
|
|
DOUT << "TailDuplication[" << SourceBlock->getParent()->getName()
|
|
|
|
<< "]: Eliminating branch: " << *Branch;
|
2003-06-22 20:10:28 +00:00
|
|
|
|
2004-10-06 03:27:37 +00:00
|
|
|
// See if we can avoid duplicating code by moving it up to a dominator of both
|
|
|
|
// blocks.
|
|
|
|
if (BasicBlock *DomBlock = FindObviousSharedDomOf(SourceBlock, DestBlock)) {
|
2006-11-26 09:46:52 +00:00
|
|
|
DOUT << "Found shared dominator: " << DomBlock->getName() << "\n";
|
2004-10-06 03:27:37 +00:00
|
|
|
|
|
|
|
// If there are non-phi instructions in DestBlock that have no operands
|
|
|
|
// defined in DestBlock, and if the instruction has no side effects, we can
|
|
|
|
// move the instruction to DomBlock instead of duplicating it.
|
|
|
|
BasicBlock::iterator BBI = DestBlock->begin();
|
|
|
|
while (isa<PHINode>(BBI)) ++BBI;
|
|
|
|
while (!isa<TerminatorInst>(BBI)) {
|
|
|
|
Instruction *I = BBI++;
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2004-10-06 03:27:37 +00:00
|
|
|
bool CanHoist = !I->isTrapping() && !I->mayWriteToMemory();
|
|
|
|
if (CanHoist) {
|
|
|
|
for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op)
|
|
|
|
if (Instruction *OpI = dyn_cast<Instruction>(I->getOperand(op)))
|
|
|
|
if (OpI->getParent() == DestBlock ||
|
|
|
|
(isa<InvokeInst>(OpI) && OpI->getParent() == DomBlock)) {
|
|
|
|
CanHoist = false;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
if (CanHoist) {
|
|
|
|
// Remove from DestBlock, move right before the term in DomBlock.
|
|
|
|
DestBlock->getInstList().remove(I);
|
|
|
|
DomBlock->getInstList().insert(DomBlock->getTerminator(), I);
|
2006-11-26 09:46:52 +00:00
|
|
|
DOUT << "Hoisted: " << *I;
|
2004-10-06 03:27:37 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
Okay, so there is no reasonable way for tail duplication to update SSA form,
as it is making effectively arbitrary modifications to the CFG and we don't
have a domset/domfrontier implementations that can handle the dynamic updates.
Instead of having a bunch of code that doesn't actually work in practice,
just demote any potentially tricky values to the stack (causing the problem
to go away entirely). Later invocations of mem2reg will rebuild SSA for us.
This fixes all of the major performance regressions with tail duplication
from LLVM 1.1. For example, this loop:
---
int popcount(int x) {
int result = 0;
while (x != 0) {
result = result + (x & 0x1);
x = x >> 1;
}
return result;
}
---
Used to be compiled into:
int %popcount(int %X) {
entry:
br label %loopentry
loopentry: ; preds = %entry, %no_exit
%x.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=3]
%result.1.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=2]
%tmp.1 = seteq int %x.0, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
no_exit: ; preds = %loopentry
%tmp.4 = and int %x.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0 ; <int> [#uses=1]
%tmp.9 = shr int %x.0, ubyte 1 ; <int> [#uses=1]
br label %loopentry
loopexit: ; preds = %loopentry
ret int %result.1.0
}
And is now compiled into:
int %popcount(int %X) {
entry:
br label %no_exit
no_exit: ; preds = %entry, %no_exit
%x.0.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=2]
%result.1.0.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=1]
%tmp.4 = and int %x.0.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0.0 ; <int> [#uses=2]
%tmp.9 = shr int %x.0.0, ubyte 1 ; <int> [#uses=2]
%tmp.1 = seteq int %tmp.9, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
loopexit: ; preds = %no_exit
ret int %tmp.6
}
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@12457 91177308-0d34-0410-b5e6-96231b3b80d8
2004-03-16 23:29:09 +00:00
|
|
|
// Tail duplication can not update SSA properties correctly if the values
|
|
|
|
// defined in the duplicated tail are used outside of the tail itself. For
|
|
|
|
// this reason, we spill all values that are used outside of the tail to the
|
|
|
|
// stack.
|
|
|
|
for (BasicBlock::iterator I = DestBlock->begin(); I != DestBlock->end(); ++I)
|
2008-04-20 22:18:22 +00:00
|
|
|
if (I->isUsedOutsideOfBlock(DestBlock)) {
|
|
|
|
// We found a use outside of the tail. Create a new stack slot to
|
|
|
|
// break this inter-block usage pattern.
|
|
|
|
DemoteRegToStack(*I);
|
Okay, so there is no reasonable way for tail duplication to update SSA form,
as it is making effectively arbitrary modifications to the CFG and we don't
have a domset/domfrontier implementations that can handle the dynamic updates.
Instead of having a bunch of code that doesn't actually work in practice,
just demote any potentially tricky values to the stack (causing the problem
to go away entirely). Later invocations of mem2reg will rebuild SSA for us.
This fixes all of the major performance regressions with tail duplication
from LLVM 1.1. For example, this loop:
---
int popcount(int x) {
int result = 0;
while (x != 0) {
result = result + (x & 0x1);
x = x >> 1;
}
return result;
}
---
Used to be compiled into:
int %popcount(int %X) {
entry:
br label %loopentry
loopentry: ; preds = %entry, %no_exit
%x.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=3]
%result.1.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=2]
%tmp.1 = seteq int %x.0, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
no_exit: ; preds = %loopentry
%tmp.4 = and int %x.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0 ; <int> [#uses=1]
%tmp.9 = shr int %x.0, ubyte 1 ; <int> [#uses=1]
br label %loopentry
loopexit: ; preds = %loopentry
ret int %result.1.0
}
And is now compiled into:
int %popcount(int %X) {
entry:
br label %no_exit
no_exit: ; preds = %entry, %no_exit
%x.0.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=2]
%result.1.0.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=1]
%tmp.4 = and int %x.0.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0.0 ; <int> [#uses=2]
%tmp.9 = shr int %x.0.0, ubyte 1 ; <int> [#uses=2]
%tmp.1 = seteq int %tmp.9, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
loopexit: ; preds = %no_exit
ret int %tmp.6
}
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@12457 91177308-0d34-0410-b5e6-96231b3b80d8
2004-03-16 23:29:09 +00:00
|
|
|
}
|
|
|
|
|
2003-06-22 20:10:28 +00:00
|
|
|
// We are going to have to map operands from the original block B to the new
|
|
|
|
// copy of the block B'. If there are PHI nodes in the DestBlock, these PHI
|
|
|
|
// nodes also define part of this mapping. Loop over these PHI nodes, adding
|
|
|
|
// them to our mapping.
|
2003-06-22 20:25:27 +00:00
|
|
|
//
|
2003-06-22 20:10:28 +00:00
|
|
|
std::map<Value*, Value*> ValueMapping;
|
|
|
|
|
|
|
|
BasicBlock::iterator BI = DestBlock->begin();
|
|
|
|
bool HadPHINodes = isa<PHINode>(BI);
|
|
|
|
for (; PHINode *PN = dyn_cast<PHINode>(BI); ++BI)
|
|
|
|
ValueMapping[PN] = PN->getIncomingValueForBlock(SourceBlock);
|
|
|
|
|
|
|
|
// Clone the non-phi instructions of the dest block into the source block,
|
|
|
|
// keeping track of the mapping...
|
|
|
|
//
|
|
|
|
for (; BI != DestBlock->end(); ++BI) {
|
|
|
|
Instruction *New = BI->clone();
|
2008-04-14 17:38:21 +00:00
|
|
|
New->setName(BI->getName());
|
2003-06-22 20:10:28 +00:00
|
|
|
SourceBlock->getInstList().push_back(New);
|
|
|
|
ValueMapping[BI] = New;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now that we have built the mapping information and cloned all of the
|
|
|
|
// instructions (giving us a new terminator, among other things), walk the new
|
|
|
|
// instructions, rewriting references of old instructions to use new
|
|
|
|
// instructions.
|
|
|
|
//
|
|
|
|
BI = Branch; ++BI; // Get an iterator to the first new instruction
|
|
|
|
for (; BI != SourceBlock->end(); ++BI)
|
|
|
|
for (unsigned i = 0, e = BI->getNumOperands(); i != e; ++i)
|
|
|
|
if (Value *Remapped = ValueMapping[BI->getOperand(i)])
|
|
|
|
BI->setOperand(i, Remapped);
|
|
|
|
|
|
|
|
// Next we check to see if any of the successors of DestBlock had PHI nodes.
|
|
|
|
// If so, we need to add entries to the PHI nodes for SourceBlock now.
|
|
|
|
for (succ_iterator SI = succ_begin(DestBlock), SE = succ_end(DestBlock);
|
|
|
|
SI != SE; ++SI) {
|
|
|
|
BasicBlock *Succ = *SI;
|
2004-09-15 17:06:42 +00:00
|
|
|
for (BasicBlock::iterator PNI = Succ->begin(); isa<PHINode>(PNI); ++PNI) {
|
|
|
|
PHINode *PN = cast<PHINode>(PNI);
|
2003-06-22 20:10:28 +00:00
|
|
|
// Ok, we have a PHI node. Figure out what the incoming value was for the
|
|
|
|
// DestBlock.
|
|
|
|
Value *IV = PN->getIncomingValueForBlock(DestBlock);
|
2005-04-21 23:48:37 +00:00
|
|
|
|
2003-06-22 20:10:28 +00:00
|
|
|
// Remap the value if necessary...
|
|
|
|
if (Value *MappedIV = ValueMapping[IV])
|
|
|
|
IV = MappedIV;
|
|
|
|
PN->addIncoming(IV, SourceBlock);
|
|
|
|
}
|
|
|
|
}
|
Okay, so there is no reasonable way for tail duplication to update SSA form,
as it is making effectively arbitrary modifications to the CFG and we don't
have a domset/domfrontier implementations that can handle the dynamic updates.
Instead of having a bunch of code that doesn't actually work in practice,
just demote any potentially tricky values to the stack (causing the problem
to go away entirely). Later invocations of mem2reg will rebuild SSA for us.
This fixes all of the major performance regressions with tail duplication
from LLVM 1.1. For example, this loop:
---
int popcount(int x) {
int result = 0;
while (x != 0) {
result = result + (x & 0x1);
x = x >> 1;
}
return result;
}
---
Used to be compiled into:
int %popcount(int %X) {
entry:
br label %loopentry
loopentry: ; preds = %entry, %no_exit
%x.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=3]
%result.1.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=2]
%tmp.1 = seteq int %x.0, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
no_exit: ; preds = %loopentry
%tmp.4 = and int %x.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0 ; <int> [#uses=1]
%tmp.9 = shr int %x.0, ubyte 1 ; <int> [#uses=1]
br label %loopentry
loopexit: ; preds = %loopentry
ret int %result.1.0
}
And is now compiled into:
int %popcount(int %X) {
entry:
br label %no_exit
no_exit: ; preds = %entry, %no_exit
%x.0.0 = phi int [ %X, %entry ], [ %tmp.9, %no_exit ] ; <int> [#uses=2]
%result.1.0.0 = phi int [ 0, %entry ], [ %tmp.6, %no_exit ] ; <int> [#uses=1]
%tmp.4 = and int %x.0.0, 1 ; <int> [#uses=1]
%tmp.6 = add int %tmp.4, %result.1.0.0 ; <int> [#uses=2]
%tmp.9 = shr int %x.0.0, ubyte 1 ; <int> [#uses=2]
%tmp.1 = seteq int %tmp.9, 0 ; <bool> [#uses=1]
br bool %tmp.1, label %loopexit, label %no_exit
loopexit: ; preds = %no_exit
ret int %tmp.6
}
git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@12457 91177308-0d34-0410-b5e6-96231b3b80d8
2004-03-16 23:29:09 +00:00
|
|
|
|
|
|
|
// Next, remove the old branch instruction, and any PHI node entries that we
|
|
|
|
// had.
|
|
|
|
BI = Branch; ++BI; // Get an iterator to the first new instruction
|
|
|
|
DestBlock->removePredecessor(SourceBlock); // Remove entries in PHI nodes...
|
|
|
|
SourceBlock->getInstList().erase(Branch); // Destroy the uncond branch...
|
2003-06-22 20:10:28 +00:00
|
|
|
|
|
|
|
// Final step: now that we have finished everything up, walk the cloned
|
|
|
|
// instructions one last time, constant propagating and DCE'ing them, because
|
|
|
|
// they may not be needed anymore.
|
|
|
|
//
|
|
|
|
if (HadPHINodes)
|
|
|
|
while (BI != SourceBlock->end())
|
|
|
|
if (!dceInstruction(BI) && !doConstantPropagation(BI))
|
|
|
|
++BI;
|
|
|
|
|
|
|
|
++NumEliminated; // We just killed a branch!
|
|
|
|
}
|