diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 0b15d42bd0a..3721fad5c3e 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -134,7 +134,7 @@ namespace {
 
     virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF);
 
-    virtual bool CanBeFoldedBy(SDNode *N, SDNode *U);
+    virtual bool CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root);
 
 // Include the pieces autogenerated from the target description.
 #include "X86GenDAGISel.inc"
@@ -227,52 +227,54 @@ static SDNode *findFlagUse(SDNode *N) {
   return NULL;
 }
 
-static void findNonImmUse(SDNode* Use, SDNode* Def, SDNode *Ignore, bool &found,
+static void findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse,
+                          SDNode *Root, SDNode *Skip, bool &found,
                           std::set<SDNode *> &Visited) {
   if (found ||
       Use->getNodeId() > Def->getNodeId() ||
       !Visited.insert(Use).second)
     return;
 
-  for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
+  for (unsigned i = 0, e = Use->getNumOperands(); !found && i != e; ++i) {
     SDNode *N = Use->getOperand(i).Val;
-    if (N == Ignore)
+    if (N == Skip)
       continue;
-    if (N != Def) {
-      findNonImmUse(N, Def, Ignore, found, Visited);
-    } else {
+    if (N == Def) {
+      if (Use == ImmedUse)
+        continue; // Immediate use is ok.
+      if (Use == Root) {
+        assert(Use->getOpcode() == ISD::STORE ||
+               Use->getOpcode() == X86ISD::CMP);
+        continue;
+      }
       found = true;
       break;
     }
+    findNonImmUse(N, Def, ImmedUse, Root, Skip, found, Visited);
   }
 }
 
-static inline bool isNonImmUse(SDNode* Use, SDNode* Def, SDNode *Ignore=NULL) {
+/// isNonImmUse - Start searching from Root up the DAG to check is Def can
+/// be reached. Return true if that's the case. However, ignore direct uses
+/// by ImmedUse (which would be U in the example illustrated in
+/// CanBeFoldedBy) and by Root (which can happen in the store case).
+/// FIXME: to be really generic, we should allow direct use by any node
+/// that is being folded. But realisticly since we only fold loads which
+/// have one non-chain use, we only need to watch out for load/op/store
+/// and load/op/cmp case where the root (store / cmp) may reach the load via
+/// its chain operand.
+static inline bool isNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse,
+                               SDNode *Skip = NULL) {
   std::set<SDNode *> Visited;
   bool found = false;
-  for (unsigned i = 0, e = Use->getNumOperands(); i != e; ++i) {
-    SDNode *N = Use->getOperand(i).Val;
-    if (N != Def && N != Ignore) {
-      findNonImmUse(N, Def, Ignore, found, Visited);
-      if (found) break;
-    }
-  }
-
-  if (!found && Ignore) {
-    // We must be checking for reachability between Def and a flag use. Go down
-    // recursively if Use also produces a flag.
-    MVT::ValueType VT = Use->getValueType(Use->getNumValues()-1);
-    if (VT == MVT::Flag && !Use->use_empty()) {
-      SDNode *FU = findFlagUse(Use);
-      if (FU)
-        return !isNonImmUse(FU, Def, Use);
-    }
-  }
+  findNonImmUse(Root, Def, ImmedUse, Root, Skip, found, Visited);
   return found;
 }
 
 
-bool X86DAGToDAGISel::CanBeFoldedBy(SDNode *N, SDNode *U) {
+bool X86DAGToDAGISel::CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root) {
+  if (FastISel) return false;
+
   // If U use can somehow reach N through another path then U can't fold N or
   // it will create a cycle. e.g. In the following diagram, U can reach N
   // through X. If N is folded into into U, then X is both a predecessor and
@@ -285,32 +287,43 @@ bool X86DAGToDAGISel::CanBeFoldedBy(SDNode *N, SDNode *U) {
   //      /        [X]
   //      |         ^
   //     [U]--------|
-  if (!FastISel && !isNonImmUse(U, N)) {
-    // If U produces a flag, then it gets (even more) interesting. Since it
-    // would have been "glued" together with its flag use, we need to check if
-    // it might reach N:
-    //
-    //      [ N ]
-    //        ^ ^
-    //        | |
-    //       [U] \--
-    //        ^   [TF]
-    //        |    |
-    //         \  /
-    //          [FU]
-    //
-    // If FU (flag use) indirectly reach N (the load), and U fold N (call it
-    // NU), then TF is a predecessor of FU and a successor of NU. But since
-    // NU and FU are flagged together, this effectively creates a cycle.
-    MVT::ValueType VT = U->getValueType(U->getNumValues()-1);
-    if (VT == MVT::Flag && !U->use_empty()) {
-      SDNode *FU = findFlagUse(U);
-      if (FU)
-        return !isNonImmUse(FU, N, U);
+
+  if (isNonImmUse(Root, N, U))
+    return false;
+
+  // If U produces a flag, then it gets (even more) interesting. Since it
+  // would have been "glued" together with its flag use, we need to check if
+  // it might reach N:
+  //
+  //       [ N ]
+  //        ^ ^
+  //        | |
+  //       [U] \--
+  //        ^   [TF]
+  //        |    ^
+  //        |    |
+  //         \  /
+  //          [FU]
+  //
+  // If FU (flag use) indirectly reach N (the load), and U fold N (call it
+  // NU), then TF is a predecessor of FU and a successor of NU. But since
+  // NU and FU are flagged together, this effectively creates a cycle.
+  bool HasFlagUse = false;
+  MVT::ValueType VT = Root->getValueType(Root->getNumValues()-1);
+  while ((VT == MVT::Flag && !Root->use_empty())) {
+    SDNode *FU = findFlagUse(Root);
+    if (FU == NULL)
+      break;
+    else {
+      Root = FU;
+      HasFlagUse = true;
     }
-    return true;
+    VT = Root->getValueType(Root->getNumValues()-1);
   }
-  return false;
+
+  if (HasFlagUse)
+    return !isNonImmUse(Root, N, Root, U);
+  return true;
 }
 
 /// MoveBelowTokenFactor - Replace TokenFactor operand with load's chain operand
@@ -909,7 +922,7 @@ bool X86DAGToDAGISel::TryFoldLoad(SDOperand P, SDOperand N,
                                   SDOperand &Index, SDOperand &Disp) {
   if (ISD::isNON_EXTLoad(N.Val) &&
       N.hasOneUse() &&
-      CanBeFoldedBy(N.Val, P.Val))
+      CanBeFoldedBy(N.Val, P.Val, P.Val))
     return SelectAddr(N.getOperand(1), Base, Scale, Index, Disp);
   return false;
 }