Teach LLVM how to scalarize packed types. Currently, this only works on

packed types with an element count of 1, although more generic support is
coming.  This allows LLVM to turn the following code:

void %foo(<1 x float> * %a) {
entry:
  %tmp1 = load <1 x float> * %a;
  %tmp2 = add <1 x float> %tmp1, %tmp1
  store <1 x float> %tmp2, <1 x float> *%a
  ret void
}

Into:

_foo:
        lfs f0, 0(r3)
        fadds f0, f0, f0
        stfs f0, 0(r3)
        blr


git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@24416 91177308-0d34-0410-b5e6-96231b3b80d8

include/llvm/CodeGen/SelectionDAG.h

@@ -257,6 +257,8 @@ public:
   ///
   SDOperand getLoad(MVT::ValueType VT, SDOperand Chain, SDOperand Ptr,
                     SDOperand SV);
+  SDOperand getVecLoad(unsigned Count, MVT::ValueType VT, SDOperand Chain, 
+                       SDOperand Ptr, SDOperand SV);
   SDOperand getExtLoad(unsigned Opcode, MVT::ValueType VT, SDOperand Chain,
                        SDOperand Ptr, SDOperand SV, MVT::ValueType EVT);
 

include/llvm/CodeGen/SelectionDAGNodes.h

@@ -107,12 +107,17 @@ namespace ISD {
     // big.  Like EXTRACT_ELEMENT, this can only be used before legalization.
     BUILD_PAIR,
 
-
     // Simple integer binary arithmetic operators.
     ADD, SUB, MUL, SDIV, UDIV, SREM, UREM,
     
     // Simple binary floating point operators.
     FADD, FSUB, FMUL, FDIV, FREM,
+    
+    // Simple abstract vector operators.  Unlike the integer and floating point
+    // binary operators, these nodes also take two additional operands:
+    // a constant element count, and a value type node indicating the type of
+    // the elements.  The order is count, type, op0, op1.
+    VADD, VSUB, VMUL,
 
     // MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing
     // an unsigned/signed value of type i[2*n], then return the top part.
@@ -209,6 +214,11 @@ namespace ISD {
     // operand, then the same operands as an LLVM load/store instruction, then a
     // SRCVALUE node that provides alias analysis information.
     LOAD, STORE,
+    
+    // Abstract vector version of LOAD.  VLOAD has a token chain as the first
+    // operand, followed by a pointer operand, a constant element count, a value
+    // type node indicating the type of the elements, and a SRCVALUE node.
+    VLOAD,
 
     // EXTLOAD, SEXTLOAD, ZEXTLOAD - These three operators all load a value from
     // memory and extend them to a larger value (e.g. load a byte into a word

lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

@@ -149,6 +149,15 @@ private:
 };
 }
 
+static unsigned scalarizedOpcode(unsigned VecOp, MVT::ValueType VT) {
+  switch (VecOp) {
+  default: assert(0 && "Don't know how to scalarize this opcode!");
+    break;
+  case ISD::VADD: return MVT::isInteger(VT) ? ISD::ADD : ISD::FADD;
+  case ISD::VSUB: return MVT::isInteger(VT) ? ISD::SUB : ISD::FSUB;
+  case ISD::VMUL: return MVT::isInteger(VT) ? ISD::MUL : ISD::FMUL;
+  }
+}
 
 SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
   : TLI(dag.getTargetLoweringInfo()), DAG(dag),
@@ -914,7 +923,27 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
     AddLegalizedOperand(SDOperand(Node, 0), Result);
     AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
     return Result.getValue(Op.ResNo);
-
+    
+  case ISD::VLOAD:
+    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
+    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
+ 
+    // If we just have one element, scalarize the result.  Otherwise, check to
+    // see if we support this operation on this type at this width.  If not,
+    // split the vector in half and try again.
+    if (1 == cast<ConstantSDNode>(Node->getOperand(2))->getValue()) {
+      MVT::ValueType SVT = cast<VTSDNode>(Node->getOperand(3))->getVT();
+      Result = LegalizeOp(DAG.getLoad(SVT, Tmp1, Tmp2, Node->getOperand(4)));
+    } else {
+      assert(0 && "Expand case for vectors unimplemented");
+    }
+    
+    // Since loads produce two values, make sure to remember that we legalized
+    // both of them.
+    AddLegalizedOperand(SDOperand(Node, 0), Result);
+    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
+    return Result.getValue(Op.ResNo);
+    
   case ISD::EXTLOAD:
   case ISD::SEXTLOAD:
   case ISD::ZEXTLOAD: {
@@ -1654,6 +1683,28 @@ SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
       Result = DAG.getNode(Node->getOpcode(), Node->getValueType(0), Tmp1,Tmp2);
     break;
 
+    // Vector binary operators
+  case ISD::VADD:
+  case ISD::VSUB:
+  case ISD::VMUL: {
+    Tmp1 = Node->getOperand(0); // Element Count
+    Tmp2 = Node->getOperand(1); // Element Type
+
+    // If we just have one element, scalarize the result.  Otherwise, check to
+    // see if we support this operation on this type at this width.  If not,
+    // split the vector in half and try again.
+    if (1 == cast<ConstantSDNode>(Tmp1)->getValue()) {
+      MVT::ValueType SVT = cast<VTSDNode>(Tmp2)->getVT();
+  
+      Result = DAG.getNode(scalarizedOpcode(Node->getOpcode(), SVT), SVT,
+                           LegalizeOp(Node->getOperand(2)),
+                           LegalizeOp(Node->getOperand(3)));
+    } else {
+      assert(0 && "Expand case for vectors unimplemented");
+    }
+    break;
+  }
+    
   case ISD::BUILD_PAIR: {
     MVT::ValueType PairTy = Node->getValueType(0);
     // TODO: handle the case where the Lo and Hi operands are not of legal type

lib/CodeGen/SelectionDAG/SelectionDAG.cpp

@@ -1092,6 +1092,23 @@ SDOperand SelectionDAG::getLoad(MVT::ValueType VT,
   return SDOperand(N, 0);
 }
 
+SDOperand SelectionDAG::getVecLoad(unsigned Count, MVT::ValueType EVT,
+                                   SDOperand Chain, SDOperand Ptr,
+                                   SDOperand SV) {
+  SDNode *&N = Loads[std::make_pair(Ptr, std::make_pair(Chain, EVT))];
+  if (N) return SDOperand(N, 0);
+  std::vector<SDOperand> Ops;
+  Ops.reserve(5);
+  Ops.push_back(Chain);
+  Ops.push_back(Ptr);
+  Ops.push_back(getConstant(Count, MVT::i32));
+  Ops.push_back(getValueType(EVT));
+  Ops.push_back(SV);
+  std::vector<MVT::ValueType> VTs;
+  VTs.reserve(2);
+  VTs.push_back(EVT); VTs.push_back(MVT::Other);  // Add token chain.
+  return getNode(ISD::VLOAD, VTs, Ops);
+}
 
 SDOperand SelectionDAG::getExtLoad(unsigned Opcode, MVT::ValueType VT,
                                    SDOperand Chain, SDOperand Ptr, SDOperand SV,
@@ -1677,6 +1694,9 @@ const char *SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::FMUL:   return "fmul";
   case ISD::FDIV:   return "fdiv";
   case ISD::FREM:   return "frem";
+  case ISD::VADD:   return "vadd";
+  case ISD::VSUB:   return "vsub";
+  case ISD::VMUL:   return "vmul";
     
   case ISD::SETCC:       return "setcc";
   case ISD::SELECT:      return "select";
@@ -1717,6 +1737,7 @@ const char *SDNode::getOperationName(const SelectionDAG *G) const {
     // Other operators
   case ISD::LOAD:    return "load";
   case ISD::STORE:   return "store";
+  case ISD::VLOAD:   return "vload";
   case ISD::EXTLOAD:    return "extload";
   case ISD::SEXTLOAD:   return "sextload";
   case ISD::ZEXTLOAD:   return "zextload";

lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp

@@ -347,40 +347,26 @@ public:
   void visitUnwind(UnwindInst &I) { assert(0 && "TODO"); }
 
   //
-  void visitBinary(User &I, unsigned Opcode);
+  void visitBinary(User &I, unsigned IntOp, unsigned FPOp, unsigned VecOp);
   void visitShift(User &I, unsigned Opcode);
-  void visitAdd(User &I) {
-    visitBinary(I, I.getType()->isFloatingPoint() ? ISD::FADD : ISD::ADD);
+  void visitAdd(User &I) { 
+    visitBinary(I, ISD::ADD, ISD::FADD, ISD::VADD); 
   }
   void visitSub(User &I);
-  void visitMul(User &I) {
-    visitBinary(I, I.getType()->isFloatingPoint() ? ISD::FMUL : ISD::MUL);
+  void visitMul(User &I) { 
+    visitBinary(I, ISD::MUL, ISD::FMUL, ISD::VMUL); 
   }
   void visitDiv(User &I) {
-    unsigned Opc;
     const Type *Ty = I.getType();
-    if (Ty->isFloatingPoint())
-      Opc = ISD::FDIV;
-    else if (Ty->isUnsigned())
-      Opc = ISD::UDIV;
-    else
-      Opc = ISD::SDIV;
-    visitBinary(I, Opc);
+    visitBinary(I, Ty->isSigned() ? ISD::SDIV : ISD::UDIV, ISD::FDIV, 0);
   }
   void visitRem(User &I) {
-    unsigned Opc;
     const Type *Ty = I.getType();
-    if (Ty->isFloatingPoint())
-      Opc = ISD::FREM;
-    else if (Ty->isUnsigned())
-      Opc = ISD::UREM;
-    else
-      Opc = ISD::SREM;
-    visitBinary(I, Opc);
+    visitBinary(I, Ty->isSigned() ? ISD::SREM : ISD::UREM, ISD::FREM, 0);
   }
-  void visitAnd(User &I) { visitBinary(I, ISD::AND); }
-  void visitOr (User &I) { visitBinary(I, ISD::OR); }
-  void visitXor(User &I) { visitBinary(I, ISD::XOR); }
+  void visitAnd(User &I) { visitBinary(I, ISD::AND, 0, 0); }
+  void visitOr (User &I) { visitBinary(I, ISD::OR,  0, 0); }
+  void visitXor(User &I) { visitBinary(I, ISD::XOR, 0, 0); }
   void visitShl(User &I) { visitShift(I, ISD::SHL); }
   void visitShr(User &I) { 
     visitShift(I, I.getType()->isUnsigned() ? ISD::SRL : ISD::SRA);
@@ -515,17 +501,26 @@ void SelectionDAGLowering::visitSub(User &I) {
         setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2));
         return;
       }
-    visitBinary(I, ISD::FSUB);
-  } else {
-    visitBinary(I, ISD::SUB);
   }
+  visitBinary(I, ISD::SUB, ISD::FSUB, ISD::VSUB);
 }
 
-void SelectionDAGLowering::visitBinary(User &I, unsigned Opcode) {
+void SelectionDAGLowering::visitBinary(User &I, unsigned IntOp, unsigned FPOp, 
+                                       unsigned VecOp) {
+  const Type *Ty = I.getType();
   SDOperand Op1 = getValue(I.getOperand(0));
   SDOperand Op2 = getValue(I.getOperand(1));
 
-  setValue(&I, DAG.getNode(Opcode, Op1.getValueType(), Op1, Op2));
+  if (Ty->isInteger()) {
+    setValue(&I, DAG.getNode(IntOp, Op1.getValueType(), Op1, Op2));
+  } else if (Ty->isFloatingPoint()) {
+    setValue(&I, DAG.getNode(FPOp, Op1.getValueType(), Op1, Op2));
+  } else {
+    const PackedType *PTy = cast<PackedType>(Ty);
+    SDOperand Num = DAG.getConstant(PTy->getNumElements(), MVT::i32);
+    SDOperand Typ = DAG.getValueType(TLI.getValueType(PTy->getElementType()));
+    setValue(&I, DAG.getNode(VecOp, Op1.getValueType(), Num, Typ, Op1, Op2));
+  }
 }
 
 void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) {
@@ -725,9 +720,19 @@ void SelectionDAGLowering::visitLoad(LoadInst &I) {
     // Do not serialize non-volatile loads against each other.
     Root = DAG.getRoot();
   }
-
-  SDOperand L = DAG.getLoad(TLI.getValueType(I.getType()), Root, Ptr,
-                            DAG.getSrcValue(I.getOperand(0)));
+  
+  const Type *Ty = I.getType();
+  SDOperand L;
+  
+  if (Type::PackedTyID == Ty->getTypeID()) {
+    const PackedType *PTy = cast<PackedType>(Ty);
+    L = DAG.getVecLoad(PTy->getNumElements(), 
+                       TLI.getValueType(PTy->getElementType()), Root, Ptr, 
+                       DAG.getSrcValue(I.getOperand(0)));
+  } else {
+    L = DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, 
+                    DAG.getSrcValue(I.getOperand(0)));
+  }
   setValue(&I, L);
 
   if (I.isVolatile())

lib/VMCore/ValueTypes.cpp

@@ -33,6 +33,7 @@ const char *MVT::getValueTypeString(MVT::ValueType VT) {
   case MVT::isVoid:return "isVoid";
   case MVT::Other: return "ch";
   case MVT::Flag:  return "flag";
+  case MVT::Vector:return "vec";
   }
 }