Add a speculative execution pass

Summary:
This is a pass for speculative execution of instructions for simple if-then (triangle) control flow. It's aimed at GPUs, but could perhaps be used in other contexts. Enabling this pass gives us a 1.0% geomean improvement on Google benchmark suites, with one benchmark improving 33%.

Credit goes to Jingyue Wu for writing an earlier version of this pass.

Patched by Bjarke Roune. 

Test Plan:
This patch adds a set of tests in test/Transforms/SpeculativeExecution/spec.ll
The pass is controlled by a flag which defaults to having the pass not run.

Reviewers: eliben, dberlin, meheff, jingyue, hfinkel

Reviewed By: jingyue, hfinkel

Subscribers: majnemer, jholewinski, llvm-commits

Differential Revision: http://reviews.llvm.org/D9360

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@237459 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/IR/BasicBlock.h

@@ -206,9 +206,19 @@ public:
     return const_cast<BasicBlock*>(this)->getUniquePredecessor();
   }
 
-  /// Return the successor of this block if it has a unique successor.
-  /// Otherwise return a null pointer.  This method is analogous to
-  /// getUniquePredeccessor above.
+  /// \brief Return the successor of this block if it has a single successor.
+  /// Otherwise return a null pointer.
+  ///
+  /// This method is analogous to getSinglePredecessor above.
+  BasicBlock *getSingleSuccessor();
+  const BasicBlock *getSingleSuccessor() const {
+    return const_cast<BasicBlock*>(this)->getSingleSuccessor();
+  }
+
+  /// \brief Return the successor of this block if it has a unique successor.
+  /// Otherwise return a null pointer.
+  ///
+  /// This method is analogous to getUniquePredecessor above.
   BasicBlock *getUniqueSuccessor();
   const BasicBlock *getUniqueSuccessor() const {
     return const_cast<BasicBlock*>(this)->getUniqueSuccessor();

include/llvm/InitializePasses.h

@@ -256,6 +256,7 @@ void initializeSinkingPass(PassRegistry&);
 void initializeSeparateConstOffsetFromGEPPass(PassRegistry &);
 void initializeSlotIndexesPass(PassRegistry&);
 void initializeSpillPlacementPass(PassRegistry&);
+void initializeSpeculativeExecutionPass(PassRegistry&);
 void initializeStackProtectorPass(PassRegistry&);
 void initializeStackColoringPass(PassRegistry&);
 void initializeStackSlotColoringPass(PassRegistry&);

include/llvm/LinkAllPasses.h

@@ -170,6 +170,7 @@ namespace {
       (void) llvm::createPartiallyInlineLibCallsPass();
       (void) llvm::createScalarizerPass();
       (void) llvm::createSeparateConstOffsetFromGEPPass();
+      (void) llvm::createSpeculativeExecutionPass();
       (void) llvm::createRewriteSymbolsPass();
       (void) llvm::createStraightLineStrengthReducePass();
       (void) llvm::createMemDerefPrinter();

include/llvm/Transforms/Scalar.h

@@ -421,6 +421,13 @@ FunctionPass *
 createSeparateConstOffsetFromGEPPass(const TargetMachine *TM = nullptr,
                                      bool LowerGEP = false);
 
+//===----------------------------------------------------------------------===//
+//
+// SpeculativeExecution - Aggressively hoist instructions to enable
+// speculative execution on targets where branches are expensive.
+//
+FunctionPass *createSpeculativeExecutionPass();
+
 //===----------------------------------------------------------------------===//
 //
 // LoadCombine - Combine loads into bigger loads.

lib/IR/BasicBlock.cpp

@@ -238,6 +238,14 @@ BasicBlock *BasicBlock::getUniquePredecessor() {
   return PredBB;
 }
 
+BasicBlock *BasicBlock::getSingleSuccessor() {
+  succ_iterator SI = succ_begin(this), E = succ_end(this);
+  if (SI == E) return nullptr; // no successors
+  BasicBlock *TheSucc = *SI;
+  ++SI;
+  return (SI == E) ? TheSucc : nullptr /* multiple successors */;
+}
+
 BasicBlock *BasicBlock::getUniqueSuccessor() {
   succ_iterator SI = succ_begin(this), E = succ_end(this);
   if (SI == E) return NULL; // No successors

lib/Transforms/IPO/PassManagerBuilder.cpp

@@ -231,6 +231,7 @@ void PassManagerBuilder::populateModulePassManager(
     MPM.add(createSROAPass(/*RequiresDomTree*/ false));
   else
     MPM.add(createScalarReplAggregatesPass(-1, false));
+
   MPM.add(createEarlyCSEPass());              // Catch trivial redundancies
   MPM.add(createJumpThreadingPass());         // Thread jumps.
   MPM.add(createCorrelatedValuePropagationPass()); // Propagate conditionals

lib/Transforms/Scalar/CMakeLists.txt

@@ -45,6 +45,7 @@ add_llvm_library(LLVMScalarOpts
   SeparateConstOffsetFromGEP.cpp
   SimplifyCFGPass.cpp
   Sink.cpp
+  SpeculativeExecution.cpp
   StraightLineStrengthReduce.cpp
   StructurizeCFG.cpp
   TailRecursionElimination.cpp

lib/Transforms/Scalar/Scalar.cpp

@@ -74,6 +74,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeSinkingPass(Registry);
   initializeTailCallElimPass(Registry);
   initializeSeparateConstOffsetFromGEPPass(Registry);
+  initializeSpeculativeExecutionPass(Registry);
   initializeStraightLineStrengthReducePass(Registry);
   initializeLoadCombinePass(Registry);
   initializePlaceBackedgeSafepointsImplPass(Registry);

lib/Transforms/Scalar/SpeculativeExecution.cpp

@@ -0,0 +1,232 @@
+//===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass hoists instructions to enable speculative execution on
+// targets where branches are expensive. This is aimed at GPUs. It
+// currently works on simple if-then and if-then-else
+// patterns.
+//
+// Removing branches is not the only motivation for this
+// pass. E.g. consider this code and assume that there is no
+// addressing mode for multiplying by sizeof(*a):
+//
+//   if (b > 0)
+//     c = a[i + 1]
+//   if (d > 0)
+//     e = a[i + 2]
+//
+// turns into
+//
+//   p = &a[i + 1];
+//   if (b > 0)
+//     c = *p;
+//   q = &a[i + 2];
+//   if (d > 0)
+//     e = *q;
+//
+// which could later be optimized to
+//
+//   r = &a[i];
+//   if (b > 0)
+//     c = r[1];
+//   if (d > 0)
+//     e = r[2];
+//
+// Later passes sink back much of the speculated code that did not enable
+// further optimization.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "speculative-execution"
+
+// The risk that speculation will not pay off increases with the
+// number of instructions speculated, so we put a limit on that.
+static cl::opt<unsigned> SpecExecMaxSpeculationCost(
+    "spec-exec-max-speculation-cost", cl::init(7), cl::Hidden,
+    cl::desc("Speculative execution is not applied to basic blocks where "
+             "the cost of the instructions to speculatively execute "
+             "exceeds this limit."));
+
+// Speculating just a few instructions from a larger block tends not
+// to be profitable and this limit prevents that. A reason for that is
+// that small basic blocks are more likely to be candidates for
+// further optimization.
+static cl::opt<unsigned> SpecExecMaxNotHoisted(
+    "spec-exec-max-not-hoisted", cl::init(5), cl::Hidden,
+    cl::desc("Speculative execution is not applied to basic blocks where the "
+             "number of instructions that would not be speculatively executed "
+             "exceeds this limit."));
+
+class SpeculativeExecution : public FunctionPass {
+ public:
+  static char ID;
+  SpeculativeExecution(): FunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnFunction(Function &F) override;
+
+ private:
+  bool runOnBasicBlock(BasicBlock &B);
+  bool considerHoistingFromTo(BasicBlock &FromBlock, BasicBlock &ToBlock);
+
+  const TargetTransformInfo *TTI = nullptr;
+};
+
+char SpeculativeExecution::ID = 0;
+INITIALIZE_PASS_BEGIN(SpeculativeExecution, "speculative-execution",
+                      "Speculatively execute instructions", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_END(SpeculativeExecution, "speculative-execution",
+                      "Speculatively execute instructions", false, false)
+
+void SpeculativeExecution::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<TargetTransformInfoWrapperPass>();
+}
+
+bool SpeculativeExecution::runOnFunction(Function &F) {
+  if (skipOptnoneFunction(F))
+    return false;
+
+  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+
+  bool Changed = false;
+  for (auto& B : F) {
+    Changed |= runOnBasicBlock(B);
+  }
+  return Changed;
+}
+
+bool SpeculativeExecution::runOnBasicBlock(BasicBlock &B) {
+  BranchInst *BI = dyn_cast<BranchInst>(B.getTerminator());
+  if (BI == nullptr)
+    return false;
+
+  if (BI->getNumSuccessors() != 2)
+    return false;
+  BasicBlock &Succ0 = *BI->getSuccessor(0);
+  BasicBlock &Succ1 = *BI->getSuccessor(1);
+
+  if (&B == &Succ0 || &B == &Succ1 || &Succ0 == &Succ1) {
+    return false;
+  }
+
+  // Hoist from if-then (triangle).
+  if (Succ0.getSinglePredecessor() != nullptr &&
+      Succ0.getSingleSuccessor() == &Succ1) {
+    return considerHoistingFromTo(Succ0, B);
+  }
+
+  // Hoist from if-else (triangle).
+  if (Succ1.getSinglePredecessor() != nullptr &&
+      Succ1.getSingleSuccessor() == &Succ0) {
+    return considerHoistingFromTo(Succ1, B);
+  }
+
+  // Hoist from if-then-else (diamond), but only if it is equivalent to
+  // an if-else or if-then due to one of the branches doing nothing.
+  if (Succ0.getSinglePredecessor() != nullptr &&
+      Succ1.getSinglePredecessor() != nullptr &&
+      Succ1.getSingleSuccessor() != nullptr &&
+      Succ1.getSingleSuccessor() != &B &&
+      Succ1.getSingleSuccessor() == Succ0.getSingleSuccessor()) {
+    // If a block has only one instruction, then that is a terminator
+    // instruction so that the block does nothing. This does happen.
+    if (Succ1.size() == 1) // equivalent to if-then
+      return considerHoistingFromTo(Succ0, B);
+    if (Succ0.size() == 1) // equivalent to if-else
+      return considerHoistingFromTo(Succ1, B);
+  }
+
+  return false;
+}
+
+static unsigned ComputeSpeculationCost(const Instruction *I,
+                                       const TargetTransformInfo &TTI) {
+  switch (Operator::getOpcode(I)) {
+    case Instruction::GetElementPtr:
+    case Instruction::Add:
+    case Instruction::Mul:
+    case Instruction::And:
+    case Instruction::Or:
+    case Instruction::Select:
+    case Instruction::Shl:
+    case Instruction::Sub:
+    case Instruction::LShr:
+    case Instruction::AShr:
+    case Instruction::Xor:
+    case Instruction::ZExt:
+    case Instruction::SExt:
+      return TTI.getUserCost(I);
+
+    default:
+      return UINT_MAX; // Disallow anything not whitelisted.
+  }
+}
+
+bool SpeculativeExecution::considerHoistingFromTo(BasicBlock &FromBlock,
+                                                  BasicBlock &ToBlock) {
+  SmallSet<const Instruction *, 8> NotHoisted;
+  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](User *U) {
+    for (Value* V : U->operand_values()) {
+      if (Instruction *I = dyn_cast<Instruction>(V)) {
+        if (NotHoisted.count(I) > 0)
+          return false;
+      }
+    }
+    return true;
+  };
+
+  unsigned TotalSpeculationCost = 0;
+  for (auto& I : FromBlock) {
+    const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
+    if (Cost != UINT_MAX && isSafeToSpeculativelyExecute(&I) &&
+        AllPrecedingUsesFromBlockHoisted(&I)) {
+      TotalSpeculationCost += Cost;
+      if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
+        return false;  // too much to hoist
+    } else {
+      NotHoisted.insert(&I);
+      if (NotHoisted.size() > SpecExecMaxNotHoisted)
+        return false; // too much left behind
+    }
+  }
+
+  if (TotalSpeculationCost == 0)
+    return false; // nothing to hoist
+
+  for (auto I = FromBlock.begin(); I != FromBlock.end();) {
+    // We have to increment I before moving Current as moving Current
+    // changes the list that I is iterating through.
+    auto Current = I;
+    ++I;
+    if (!NotHoisted.count(Current)) {
+      Current->moveBefore(ToBlock.getTerminator());
+    }
+  }
+  return true;
+}
+
+namespace llvm {
+
+FunctionPass *createSpeculativeExecutionPass() {
+  return new SpeculativeExecution();
+}
+
+}  // namespace llvm

test/Transforms/SpeculativeExecution/spec.ll

@@ -0,0 +1,195 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN:   -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN:   | FileCheck %s
+
+target datalayout = "e-i64:64-v16:16-v32:32-n16:32:64"
+
+; Hoist in if-then pattern.
+define void @ifThen() {
+; CHECK-LABEL: @ifThen(
+; CHECK: %x = add i32 2, 3
+; CHECK: br i1 true
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %b
+; CHECK: b:
+b:
+; CHECK: ret void
+  ret void
+}
+
+; Hoist in if-else pattern.
+define void @ifElse() {
+; CHECK-LABEL: @ifElse(
+; CHECK: %x = add i32 2, 3
+; CHECK: br i1 true
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %b
+; CHECK: b:
+b:
+; CHECK: ret void
+  ret void
+}
+
+; Hoist in if-then-else pattern if it is equivalent to if-then.
+define void @ifElseThenAsIfThen() {
+; CHECK-LABEL: @ifElseThenAsIfThen(
+; CHECK: %x = add i32 2, 3
+; CHECK: br
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Hoist in if-then-else pattern if it is equivalent to if-else.
+define void @ifElseThenAsIfElse() {
+; CHECK-LABEL: @ifElseThenAsIfElse(
+; CHECK: %x = add i32 2, 3
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Do not hoist if-then-else pattern if it is not equivalent to if-then
+; or if-else.
+define void @ifElseThen() {
+; CHECK-LABEL: @ifElseThen(
+; CHECK: br
+  br i1 true, label %a, label %b
+; CHECK: a:
+a:
+; CHECK: %x = add
+  %x = add i32 2, 3
+; CHECK: br label
+  br label %c
+; CHECK: b:
+b:
+; CHECK: %y = add
+  %y = add i32 2, 3
+  br label %c
+; CHECK: c
+c:
+  ret void
+}
+
+; Do not hoist loads and do not hoist an instruction past a definition of
+; an operand.
+define void @doNotHoistPastDef() {
+; CHECK-LABEL: @doNotHoistPastDef(
+  br i1 true, label %b, label %a
+; CHECK-NOT: load
+; CHECK-NOT: add
+; CHECK: a:
+a:
+; CHECK: %def = load
+  %def = load i32, i32* null
+; CHECK: %use = add
+  %use = add i32 %def, 0
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Case with nothing to speculate.
+define void @nothingToSpeculate() {
+; CHECK-LABEL: @nothingToSpeculate(
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %def = load
+  %def = load i32, i32* null
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Still hoist if an operand is defined before the block or is itself hoisted.
+define void @hoistIfNotPastDef() {
+; CHECK-LABEL: @hoistIfNotPastDef(
+; CHECK: %x = load
+  %x = load i32, i32* null
+; CHECK: %y = add i32 %x, 1
+; CHECK: %z = add i32 %y, 1
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+  %y = add i32 %x, 1
+  %z = add i32 %y, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Do not hoist if the speculation cost is too high.
+define void @costTooHigh() {
+; CHECK-LABEL: @costTooHigh(
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %r1 = add
+  %r1 = add i32 1, 1
+; CHECK: %r2 = add
+  %r2 = add i32 1, 1
+; CHECK: %r3 = add
+  %r3 = add i32 1, 1
+; CHECK: %r4 = add
+  %r4 = add i32 1, 1
+; CHECK: %r5 = add
+  %r5 = add i32 1, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}
+
+; Do not hoist if too many instructions are left behind.
+define void @tooMuchLeftBehind() {
+; CHECK-LABEL: @tooMuchLeftBehind(
+; CHECK: br
+  br i1 true, label %b, label %a
+; CHECK: a:
+a:
+; CHECK: %x = load
+  %x = load i32, i32* null
+; CHECK: %r1 = add
+  %r1 = add i32 %x, 1
+; CHECK: %r2 = add
+  %r2 = add i32 %x, 1
+; CHECK: %r3 = add
+  %r3 = add i32 %x, 1
+  br label %b
+; CHECK: b:
+b:
+  ret void
+}