mirror of
https://github.com/c64scene-ar/llvm-6502.git
synced 2024-08-23 17:29:19 +00:00
X86: Properly decode shuffle masks when the constant pool type is weird
It's possible for the constant pool entry for the shuffle mask to come from a completely different operation. This occurs when Constants have the same bit pattern but have different types. Make DecodePSHUFBMask tolerant of types which, after a bitcast, are appropriately sized vector types. This fixes PR22188. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@225597 91177308-0d34-0410-b5e6-96231b3b80d8
This commit is contained in:
parent
776673ea09
commit
d2f4460ee7
@ -13,7 +13,10 @@
|
|||||||
//===----------------------------------------------------------------------===//
|
//===----------------------------------------------------------------------===//
|
||||||
|
|
||||||
#include "X86ShuffleDecode.h"
|
#include "X86ShuffleDecode.h"
|
||||||
|
#include "llvm/Analysis/ConstantFolding.h"
|
||||||
#include "llvm/IR/Constants.h"
|
#include "llvm/IR/Constants.h"
|
||||||
|
#include "llvm/IR/DataLayout.h"
|
||||||
|
#include "llvm/IR/InstrTypes.h"
|
||||||
#include "llvm/CodeGen/MachineValueType.h"
|
#include "llvm/CodeGen/MachineValueType.h"
|
||||||
|
|
||||||
//===----------------------------------------------------------------------===//
|
//===----------------------------------------------------------------------===//
|
||||||
@ -253,57 +256,64 @@ void DecodeVPERM2X128Mask(MVT VT, unsigned Imm,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) {
|
void DecodePSHUFBMask(const Constant *C, const DataLayout *TD,
|
||||||
|
SmallVectorImpl<int> &ShuffleMask) {
|
||||||
Type *MaskTy = C->getType();
|
Type *MaskTy = C->getType();
|
||||||
assert(MaskTy->isVectorTy() && "Expected a vector constant mask!");
|
// It is not an error for the PSHUFB mask to not be a vector of i8 because the
|
||||||
assert(MaskTy->getVectorElementType()->isIntegerTy(8) &&
|
// constant pool uniques constants by their bit representation.
|
||||||
"Expected i8 constant mask elements!");
|
// e.g. the following take up the same space in the constant pool:
|
||||||
int NumElements = MaskTy->getVectorNumElements();
|
// i128 -170141183420855150465331762880109871104
|
||||||
|
//
|
||||||
|
// <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
|
||||||
|
//
|
||||||
|
// <4 x i32> <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32
|
||||||
|
// -2147483648>
|
||||||
|
|
||||||
|
unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
|
||||||
|
|
||||||
|
VectorType *DestTy = nullptr;
|
||||||
|
if (MaskTySize == 128)
|
||||||
|
DestTy = VectorType::get(Type::getInt8Ty(MaskTy->getContext()), 16);
|
||||||
|
else if (MaskTySize == 256)
|
||||||
|
DestTy = VectorType::get(Type::getInt8Ty(MaskTy->getContext()), 32);
|
||||||
|
|
||||||
// FIXME: Add support for AVX-512.
|
// FIXME: Add support for AVX-512.
|
||||||
assert((NumElements == 16 || NumElements == 32) &&
|
if (!DestTy)
|
||||||
"Only 128-bit and 256-bit vectors supported!");
|
return;
|
||||||
|
|
||||||
|
if (DestTy != MaskTy) {
|
||||||
|
if (!CastInst::castIsValid(Instruction::BitCast, const_cast<Constant *>(C),
|
||||||
|
DestTy))
|
||||||
|
return;
|
||||||
|
|
||||||
|
C = ConstantFoldInstOperands(Instruction::BitCast, DestTy,
|
||||||
|
const_cast<Constant *>(C), TD);
|
||||||
|
MaskTy = DestTy;
|
||||||
|
}
|
||||||
|
|
||||||
|
int NumElements = MaskTy->getVectorNumElements();
|
||||||
ShuffleMask.reserve(NumElements);
|
ShuffleMask.reserve(NumElements);
|
||||||
|
|
||||||
if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
|
for (int i = 0; i < NumElements; ++i) {
|
||||||
assert((unsigned)NumElements == CDS->getNumElements() &&
|
// For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
|
||||||
"Constant mask has a different number of elements!");
|
// lane of the vector we're inside.
|
||||||
|
int Base = i < 16 ? 0 : 16;
|
||||||
for (int i = 0; i < NumElements; ++i) {
|
Constant *COp = C->getAggregateElement(i);
|
||||||
// For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
|
if (!COp) {
|
||||||
// lane of the vector we're inside.
|
ShuffleMask.clear();
|
||||||
int Base = i < 16 ? 0 : 16;
|
return;
|
||||||
uint64_t Element = CDS->getElementAsInteger(i);
|
} else if (isa<UndefValue>(COp)) {
|
||||||
// If the high bit (7) of the byte is set, the element is zeroed.
|
ShuffleMask.push_back(SM_SentinelUndef);
|
||||||
if (Element & (1 << 7))
|
continue;
|
||||||
ShuffleMask.push_back(SM_SentinelZero);
|
|
||||||
else {
|
|
||||||
// Only the least significant 4 bits of the byte are used.
|
|
||||||
int Index = Base + (Element & 0xf);
|
|
||||||
ShuffleMask.push_back(Index);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else if (auto *CV = dyn_cast<ConstantVector>(C)) {
|
uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
|
||||||
assert((unsigned)NumElements == CV->getNumOperands() &&
|
// If the high bit (7) of the byte is set, the element is zeroed.
|
||||||
"Constant mask has a different number of elements!");
|
if (Element & (1 << 7))
|
||||||
|
ShuffleMask.push_back(SM_SentinelZero);
|
||||||
for (int i = 0; i < NumElements; ++i) {
|
else {
|
||||||
// For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
|
// Only the least significant 4 bits of the byte are used.
|
||||||
// lane of the vector we're inside.
|
int Index = Base + (Element & 0xf);
|
||||||
int Base = i < 16 ? 0 : 16;
|
ShuffleMask.push_back(Index);
|
||||||
Constant *COp = CV->getOperand(i);
|
|
||||||
if (isa<UndefValue>(COp)) {
|
|
||||||
ShuffleMask.push_back(SM_SentinelUndef);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
uint64_t Element = cast<ConstantInt>(COp)->getZExtValue();
|
|
||||||
// If the high bit (7) of the byte is set, the element is zeroed.
|
|
||||||
if (Element & (1 << 7))
|
|
||||||
ShuffleMask.push_back(SM_SentinelZero);
|
|
||||||
else {
|
|
||||||
// Only the least significant 4 bits of the byte are used.
|
|
||||||
int Index = Base + (Element & 0xf);
|
|
||||||
ShuffleMask.push_back(Index);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -24,6 +24,7 @@
|
|||||||
|
|
||||||
namespace llvm {
|
namespace llvm {
|
||||||
class Constant;
|
class Constant;
|
||||||
|
class DataLayout;
|
||||||
class MVT;
|
class MVT;
|
||||||
|
|
||||||
enum { SM_SentinelUndef = -1, SM_SentinelZero = -2 };
|
enum { SM_SentinelUndef = -1, SM_SentinelZero = -2 };
|
||||||
@ -68,7 +69,8 @@ void DecodeUNPCKHMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
|
|||||||
void DecodeUNPCKLMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
|
void DecodeUNPCKLMask(MVT VT, SmallVectorImpl<int> &ShuffleMask);
|
||||||
|
|
||||||
/// \brief Decode a PSHUFB mask from an IR-level vector constant.
|
/// \brief Decode a PSHUFB mask from an IR-level vector constant.
|
||||||
void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask);
|
void DecodePSHUFBMask(const Constant *C, const DataLayout *TD,
|
||||||
|
SmallVectorImpl<int> &ShuffleMask);
|
||||||
|
|
||||||
/// \brief Decode a PSHUFB mask from a raw array of constants such as from
|
/// \brief Decode a PSHUFB mask from a raw array of constants such as from
|
||||||
/// BUILD_VECTOR.
|
/// BUILD_VECTOR.
|
||||||
|
@ -5365,7 +5365,7 @@ static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
|
|||||||
/// IsUnary to true if only uses one source. Note that this will set IsUnary for
|
/// IsUnary to true if only uses one source. Note that this will set IsUnary for
|
||||||
/// shuffles which use a single input multiple times, and in those cases it will
|
/// shuffles which use a single input multiple times, and in those cases it will
|
||||||
/// adjust the mask to only have indices within that single input.
|
/// adjust the mask to only have indices within that single input.
|
||||||
static bool getTargetShuffleMask(SDNode *N, MVT VT,
|
static bool getTargetShuffleMask(SDNode *N, MVT VT, const DataLayout *TD,
|
||||||
SmallVectorImpl<int> &Mask, bool &IsUnary) {
|
SmallVectorImpl<int> &Mask, bool &IsUnary) {
|
||||||
unsigned NumElems = VT.getVectorNumElements();
|
unsigned NumElems = VT.getVectorNumElements();
|
||||||
SDValue ImmN;
|
SDValue ImmN;
|
||||||
@ -5472,13 +5472,7 @@ static bool getTargetShuffleMask(SDNode *N, MVT VT,
|
|||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (auto *C = dyn_cast<Constant>(MaskCP->getConstVal())) {
|
if (auto *C = dyn_cast<Constant>(MaskCP->getConstVal())) {
|
||||||
// FIXME: Support AVX-512 here.
|
DecodePSHUFBMask(C, TD, Mask);
|
||||||
Type *Ty = C->getType();
|
|
||||||
if (!Ty->isVectorTy() || (Ty->getVectorNumElements() != 16 &&
|
|
||||||
Ty->getVectorNumElements() != 32))
|
|
||||||
return false;
|
|
||||||
|
|
||||||
DecodePSHUFBMask(C, Mask);
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -5541,6 +5535,7 @@ static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
|
|||||||
SDValue V = SDValue(N, 0);
|
SDValue V = SDValue(N, 0);
|
||||||
EVT VT = V.getValueType();
|
EVT VT = V.getValueType();
|
||||||
unsigned Opcode = V.getOpcode();
|
unsigned Opcode = V.getOpcode();
|
||||||
|
const DataLayout *TD = DAG.getSubtarget().getDataLayout();
|
||||||
|
|
||||||
// Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
|
// Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
|
||||||
if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
|
if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
|
||||||
@ -5562,7 +5557,7 @@ static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
|
|||||||
SmallVector<int, 16> ShuffleMask;
|
SmallVector<int, 16> ShuffleMask;
|
||||||
bool IsUnary;
|
bool IsUnary;
|
||||||
|
|
||||||
if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary))
|
if (!getTargetShuffleMask(N, ShufVT, TD, ShuffleMask, IsUnary))
|
||||||
return SDValue();
|
return SDValue();
|
||||||
|
|
||||||
int Elt = ShuffleMask[Index];
|
int Elt = ShuffleMask[Index];
|
||||||
@ -22117,7 +22112,8 @@ static bool combineX86ShufflesRecursively(SDValue Op, SDValue Root,
|
|||||||
return false;
|
return false;
|
||||||
SmallVector<int, 16> OpMask;
|
SmallVector<int, 16> OpMask;
|
||||||
bool IsUnary;
|
bool IsUnary;
|
||||||
bool HaveMask = getTargetShuffleMask(Op.getNode(), VT, OpMask, IsUnary);
|
bool HaveMask = getTargetShuffleMask(
|
||||||
|
Op.getNode(), VT, Subtarget->getDataLayout(), OpMask, IsUnary);
|
||||||
// We only can combine unary shuffles which we can decode the mask for.
|
// We only can combine unary shuffles which we can decode the mask for.
|
||||||
if (!HaveMask || !IsUnary)
|
if (!HaveMask || !IsUnary)
|
||||||
return false;
|
return false;
|
||||||
@ -22208,10 +22204,12 @@ static bool combineX86ShufflesRecursively(SDValue Op, SDValue Root,
|
|||||||
///
|
///
|
||||||
/// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
|
/// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
|
||||||
/// PSHUF-style masks that can be reused with such instructions.
|
/// PSHUF-style masks that can be reused with such instructions.
|
||||||
static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
|
static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N,
|
||||||
|
const DataLayout *TD) {
|
||||||
SmallVector<int, 4> Mask;
|
SmallVector<int, 4> Mask;
|
||||||
bool IsUnary;
|
bool IsUnary;
|
||||||
bool HaveMask = getTargetShuffleMask(N.getNode(), N.getSimpleValueType(), Mask, IsUnary);
|
bool HaveMask = getTargetShuffleMask(N.getNode(), N.getSimpleValueType(), TD,
|
||||||
|
Mask, IsUnary);
|
||||||
(void)HaveMask;
|
(void)HaveMask;
|
||||||
assert(HaveMask);
|
assert(HaveMask);
|
||||||
|
|
||||||
@ -22243,6 +22241,7 @@ combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
|
|||||||
assert(N.getOpcode() == X86ISD::PSHUFD &&
|
assert(N.getOpcode() == X86ISD::PSHUFD &&
|
||||||
"Called with something other than an x86 128-bit half shuffle!");
|
"Called with something other than an x86 128-bit half shuffle!");
|
||||||
SDLoc DL(N);
|
SDLoc DL(N);
|
||||||
|
const DataLayout *TD = DAG.getSubtarget().getDataLayout();
|
||||||
|
|
||||||
// Walk up a single-use chain looking for a combinable shuffle. Keep a stack
|
// Walk up a single-use chain looking for a combinable shuffle. Keep a stack
|
||||||
// of the shuffles in the chain so that we can form a fresh chain to replace
|
// of the shuffles in the chain so that we can form a fresh chain to replace
|
||||||
@ -22328,7 +22327,7 @@ combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
|
|||||||
return SDValue();
|
return SDValue();
|
||||||
|
|
||||||
// Merge this node's mask and our incoming mask.
|
// Merge this node's mask and our incoming mask.
|
||||||
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
|
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V, TD);
|
||||||
for (int &M : Mask)
|
for (int &M : Mask)
|
||||||
M = VMask[M];
|
M = VMask[M];
|
||||||
V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
|
V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
|
||||||
@ -22377,6 +22376,7 @@ static bool combineRedundantHalfShuffle(SDValue N, MutableArrayRef<int> Mask,
|
|||||||
"Called with something other than an x86 128-bit half shuffle!");
|
"Called with something other than an x86 128-bit half shuffle!");
|
||||||
SDLoc DL(N);
|
SDLoc DL(N);
|
||||||
unsigned CombineOpcode = N.getOpcode();
|
unsigned CombineOpcode = N.getOpcode();
|
||||||
|
const DataLayout *TD = DAG.getSubtarget().getDataLayout();
|
||||||
|
|
||||||
// Walk up a single-use chain looking for a combinable shuffle.
|
// Walk up a single-use chain looking for a combinable shuffle.
|
||||||
SDValue V = N.getOperand(0);
|
SDValue V = N.getOperand(0);
|
||||||
@ -22415,7 +22415,7 @@ static bool combineRedundantHalfShuffle(SDValue N, MutableArrayRef<int> Mask,
|
|||||||
|
|
||||||
// Merge this node's mask and our incoming mask (adjusted to account for all
|
// Merge this node's mask and our incoming mask (adjusted to account for all
|
||||||
// the pshufd instructions encountered).
|
// the pshufd instructions encountered).
|
||||||
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
|
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V, TD);
|
||||||
for (int &M : Mask)
|
for (int &M : Mask)
|
||||||
M = VMask[M];
|
M = VMask[M];
|
||||||
V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0),
|
V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0),
|
||||||
@ -22437,13 +22437,14 @@ static SDValue PerformTargetShuffleCombine(SDValue N, SelectionDAG &DAG,
|
|||||||
const X86Subtarget *Subtarget) {
|
const X86Subtarget *Subtarget) {
|
||||||
SDLoc DL(N);
|
SDLoc DL(N);
|
||||||
MVT VT = N.getSimpleValueType();
|
MVT VT = N.getSimpleValueType();
|
||||||
|
const DataLayout *TD = Subtarget->getDataLayout();
|
||||||
SmallVector<int, 4> Mask;
|
SmallVector<int, 4> Mask;
|
||||||
|
|
||||||
switch (N.getOpcode()) {
|
switch (N.getOpcode()) {
|
||||||
case X86ISD::PSHUFD:
|
case X86ISD::PSHUFD:
|
||||||
case X86ISD::PSHUFLW:
|
case X86ISD::PSHUFLW:
|
||||||
case X86ISD::PSHUFHW:
|
case X86ISD::PSHUFHW:
|
||||||
Mask = getPSHUFShuffleMask(N);
|
Mask = getPSHUFShuffleMask(N, TD);
|
||||||
assert(Mask.size() == 4);
|
assert(Mask.size() == 4);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
@ -22495,8 +22496,8 @@ static SDValue PerformTargetShuffleCombine(SDValue N, SelectionDAG &DAG,
|
|||||||
while (D.getOpcode() == ISD::BITCAST && D.hasOneUse())
|
while (D.getOpcode() == ISD::BITCAST && D.hasOneUse())
|
||||||
D = D.getOperand(0);
|
D = D.getOperand(0);
|
||||||
if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
|
if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
|
||||||
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
|
SmallVector<int, 4> VMask = getPSHUFShuffleMask(V, TD);
|
||||||
SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
|
SmallVector<int, 4> DMask = getPSHUFShuffleMask(D, TD);
|
||||||
int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
|
int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
|
||||||
int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
|
int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
|
||||||
int WordMask[8];
|
int WordMask[8];
|
||||||
@ -22749,9 +22750,10 @@ static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
|
|||||||
if (!InVec.hasOneUse())
|
if (!InVec.hasOneUse())
|
||||||
return SDValue();
|
return SDValue();
|
||||||
|
|
||||||
|
const DataLayout *TD = DAG.getSubtarget().getDataLayout();
|
||||||
SmallVector<int, 16> ShuffleMask;
|
SmallVector<int, 16> ShuffleMask;
|
||||||
bool UnaryShuffle;
|
bool UnaryShuffle;
|
||||||
if (!getTargetShuffleMask(InVec.getNode(), CurrentVT.getSimpleVT(),
|
if (!getTargetShuffleMask(InVec.getNode(), CurrentVT.getSimpleVT(), TD,
|
||||||
ShuffleMask, UnaryShuffle))
|
ShuffleMask, UnaryShuffle))
|
||||||
return SDValue();
|
return SDValue();
|
||||||
|
|
||||||
|
@ -1161,7 +1161,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
|
|||||||
|
|
||||||
if (auto *C = getConstantFromPool(*MI, MaskOp)) {
|
if (auto *C = getConstantFromPool(*MI, MaskOp)) {
|
||||||
SmallVector<int, 16> Mask;
|
SmallVector<int, 16> Mask;
|
||||||
DecodePSHUFBMask(C, Mask);
|
DecodePSHUFBMask(C, TM.getSubtargetImpl()->getDataLayout(), Mask);
|
||||||
if (!Mask.empty())
|
if (!Mask.empty())
|
||||||
OutStreamer.AddComment(getShuffleComment(DstOp, SrcOp, Mask));
|
OutStreamer.AddComment(getShuffleComment(DstOp, SrcOp, Mask));
|
||||||
}
|
}
|
||||||
|
@ -27,4 +27,14 @@ define <16 x i8> @test3(<16 x i8> %V) {
|
|||||||
ret <16 x i8> %1
|
ret <16 x i8> %1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
; Test that we won't crash when the constant was reused for another instruction.
|
||||||
|
|
||||||
|
define <16 x i8> @test4(<2 x i64>* %V) {
|
||||||
|
; CHECK-LABEL: test4
|
||||||
|
; CHECK: pshufb {{.*}}# xmm0 = xmm0[8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7]
|
||||||
|
store <2 x i64> <i64 1084818905618843912, i64 506097522914230528>, <2 x i64>* %V, align 16
|
||||||
|
%1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> undef, <16 x i8> <i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
|
||||||
|
ret <16 x i8> %1
|
||||||
|
}
|
||||||
|
|
||||||
declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone
|
declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone
|
||||||
|
Loading…
Reference in New Issue
Block a user