diff --git a/include/llvm/Analysis/LoopAccessAnalysis.h b/include/llvm/Analysis/LoopAccessAnalysis.h index 941b50a2b25..471959855aa 100644 --- a/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/include/llvm/Analysis/LoopAccessAnalysis.h @@ -60,6 +60,24 @@ public: const Loop *TheLoop); }; +/// \brief Collection of parameters shared beetween the Loop Vectorizer and the +/// Loop Access Analysis. +struct VectorizerParams { + /// \brief Maximum SIMD width. + static const unsigned MaxVectorWidth; + + /// \brief VF as overridden by the user. + static unsigned VectorizationFactor; + /// \brief Interleave factor as overridden by the user. + static unsigned VectorizationInterleave; + /// \brief True if force-vector-interleave was specified by the user. + static bool isInterleaveForced(); + + /// \\brief When performing memory disambiguation checks at runtime do not + /// make more than this number of comparisons. + static const unsigned RuntimeMemoryCheckThreshold; +}; + /// \brief Drive the analysis of memory accesses in the loop /// /// This class is responsible for analyzing the memory accesses of a loop. It @@ -76,30 +94,6 @@ public: /// RuntimePointerCheck class. class LoopAccessInfo { public: - /// \brief Collection of parameters used from the vectorizer. - struct VectorizerParams { - /// \brief Maximum simd width. - unsigned MaxVectorWidth; - - /// \brief VF as overridden by the user. - unsigned VectorizationFactor; - /// \brief Interleave factor as overridden by the user. - unsigned VectorizationInterleave; - - /// \\brief When performing memory disambiguation checks at runtime do not - /// make more than this number of comparisons. - unsigned RuntimeMemoryCheckThreshold; - - VectorizerParams(unsigned MaxVectorWidth, - unsigned VectorizationFactor, - unsigned VectorizationInterleave, - unsigned RuntimeMemoryCheckThreshold) : - MaxVectorWidth(MaxVectorWidth), - VectorizationFactor(VectorizationFactor), - VectorizationInterleave(VectorizationInterleave), - RuntimeMemoryCheckThreshold(RuntimeMemoryCheckThreshold) {} - }; - /// This struct holds information about the memory runtime legality check that /// a group of pointers do not overlap. struct RuntimePointerCheck { @@ -143,11 +137,9 @@ public: LoopAccessInfo(Function *F, Loop *L, ScalarEvolution *SE, const DataLayout *DL, const TargetLibraryInfo *TLI, - AliasAnalysis *AA, DominatorTree *DT, - const VectorizerParams &VectParams) : + AliasAnalysis *AA, DominatorTree *DT) : TheFunction(F), TheLoop(L), SE(SE), DL(DL), TLI(TLI), AA(AA), DT(DT), - NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1U), - VectParams(VectParams) {} + NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1U) {} /// Return true we can analyze the memory accesses in the loop and there are /// no memory dependence cycles. Replaces symbolic strides using Strides. @@ -192,9 +184,6 @@ private: unsigned NumStores; unsigned MaxSafeDepDistBytes; - - /// \brief Vectorizer parameters used by the analysis. - VectorizerParams VectParams; }; Value *stripIntegerCast(Value *V); diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp index 35c5807be08..c0cad5c0894 100644 --- a/lib/Analysis/LoopAccessAnalysis.cpp +++ b/lib/Analysis/LoopAccessAnalysis.cpp @@ -25,6 +25,31 @@ using namespace llvm; #define DEBUG_TYPE "loop-vectorize" +static cl::opt +VectorizationFactor("force-vector-width", cl::Hidden, + cl::desc("Sets the SIMD width. Zero is autoselect."), + cl::location(VectorizerParams::VectorizationFactor)); +unsigned VectorizerParams::VectorizationFactor = 0; + +static cl::opt +VectorizationInterleave("force-vector-interleave", cl::Hidden, + cl::desc("Sets the vectorization interleave count. " + "Zero is autoselect."), + cl::location( + VectorizerParams::VectorizationInterleave)); +unsigned VectorizerParams::VectorizationInterleave = 0; + +/// When performing memory disambiguation checks at runtime do not make more +/// than this number of comparisons. +const unsigned VectorizerParams::RuntimeMemoryCheckThreshold = 8; + +/// Maximum SIMD width. +const unsigned VectorizerParams::MaxVectorWidth = 64; + +bool VectorizerParams::isInterleaveForced() { + return ::VectorizationInterleave.getNumOccurrences() > 0; +} + void VectorizationReport::emitAnalysis(VectorizationReport &Message, const Function *TheFunction, const Loop *TheLoop) { @@ -454,10 +479,9 @@ public: typedef PointerIntPair MemAccessInfo; typedef SmallPtrSet MemAccessInfoSet; - MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L, - const LoopAccessInfo::VectorizerParams &VectParams) + MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L) : SE(Se), DL(Dl), InnermostLoop(L), AccessIdx(0), - ShouldRetryWithRuntimeCheck(false), VectParams(VectParams) {} + ShouldRetryWithRuntimeCheck(false) {} /// \brief Register the location (instructions are given increasing numbers) /// of a write access. @@ -512,9 +536,6 @@ private: /// vectorize this loop with runtime checks. bool ShouldRetryWithRuntimeCheck; - /// \brief Vectorizer parameters used by the analysis. - LoopAccessInfo::VectorizerParams VectParams; - /// \brief Check whether there is a plausible dependence between the two /// accesses. /// @@ -638,7 +659,8 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance, // Store-load forwarding distance. const unsigned NumCyclesForStoreLoadThroughMemory = 8*TypeByteSize; // Maximum vector factor. - unsigned MaxVFWithoutSLForwardIssues = VectParams.MaxVectorWidth*TypeByteSize; + unsigned MaxVFWithoutSLForwardIssues = + VectorizerParams::MaxVectorWidth * TypeByteSize; if(MaxSafeDepDistBytes < MaxVFWithoutSLForwardIssues) MaxVFWithoutSLForwardIssues = MaxSafeDepDistBytes; @@ -657,7 +679,8 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance, } if (MaxVFWithoutSLForwardIssues < MaxSafeDepDistBytes && - MaxVFWithoutSLForwardIssues != VectParams.MaxVectorWidth*TypeByteSize) + MaxVFWithoutSLForwardIssues != + VectorizerParams::MaxVectorWidth * TypeByteSize) MaxSafeDepDistBytes = MaxVFWithoutSLForwardIssues; return false; } @@ -762,10 +785,10 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, unsigned Distance = (unsigned) Val.getZExtValue(); // Bail out early if passed-in parameters make vectorization not feasible. - unsigned ForcedFactor = (VectParams.VectorizationFactor ? - VectParams.VectorizationFactor : 1); - unsigned ForcedUnroll = (VectParams.VectorizationInterleave ? - VectParams.VectorizationInterleave : 1); + unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ? + VectorizerParams::VectorizationFactor : 1); + unsigned ForcedUnroll = (VectorizerParams::VectorizationInterleave ? + VectorizerParams::VectorizationInterleave : 1); // The distance must be bigger than the size needed for a vectorized version // of the operation and the size of the vectorized operation must not be @@ -848,7 +871,7 @@ bool LoopAccessInfo::canVectorizeMemory(ValueToValueMap &Strides) { PtrRtCheck.Need = false; const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel(); - MemoryDepChecker DepChecker(SE, DL, TheLoop, VectParams); + MemoryDepChecker DepChecker(SE, DL, TheLoop); // For each block. for (Loop::block_iterator bb = TheLoop->block_begin(), @@ -1017,7 +1040,8 @@ bool LoopAccessInfo::canVectorizeMemory(ValueToValueMap &Strides) { // Check that we did not collect too many pointers or found an unsizeable // pointer. - if (!CanDoRT || NumComparisons > VectParams.RuntimeMemoryCheckThreshold) { + if (!CanDoRT || + NumComparisons > VectorizerParams::RuntimeMemoryCheckThreshold) { PtrRtCheck.reset(); CanDoRT = false; } @@ -1057,14 +1081,15 @@ bool LoopAccessInfo::canVectorizeMemory(ValueToValueMap &Strides) { TheLoop, Strides, true); // Check that we did not collect too many pointers or found an unsizeable // pointer. - if (!CanDoRT || NumComparisons > VectParams.RuntimeMemoryCheckThreshold) { + if (!CanDoRT || + NumComparisons > VectorizerParams::RuntimeMemoryCheckThreshold) { if (!CanDoRT && NumComparisons > 0) emitAnalysis(VectorizationReport() << "cannot check memory dependencies at runtime"); else emitAnalysis(VectorizationReport() << NumComparisons << " exceeds limit of " - << VectParams.RuntimeMemoryCheckThreshold + << VectorizerParams::RuntimeMemoryCheckThreshold << " dependent memory operations checked at runtime"); DEBUG(dbgs() << "LV: Can't vectorize with memory checks\n"); PtrRtCheck.reset(); diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp index 1ddb45e5df6..f53c913c647 100644 --- a/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -106,15 +106,6 @@ using namespace llvm::PatternMatch; STATISTIC(LoopsVectorized, "Number of loops vectorized"); STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization"); -static cl::opt -VectorizationFactor("force-vector-width", cl::init(0), cl::Hidden, - cl::desc("Sets the SIMD width. Zero is autoselect.")); - -static cl::opt -VectorizationInterleave("force-vector-interleave", cl::init(0), cl::Hidden, - cl::desc("Sets the vectorization interleave count. " - "Zero is autoselect.")); - static cl::opt EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden, cl::desc("Enable if-conversion during vectorization.")); @@ -145,13 +136,6 @@ static cl::opt EnableMemAccessVersioning( /// We don't unroll loops with a known constant trip count below this number. static const unsigned TinyTripCountUnrollThreshold = 128; -/// When performing memory disambiguation checks at runtime do not make more -/// than this number of comparisons. -static const unsigned RuntimeMemoryCheckThreshold = 8; - -/// Maximum simd width. -static const unsigned MaxVectorWidth = 64; - static cl::opt ForceTargetNumScalarRegs( "force-target-num-scalar-regs", cl::init(0), cl::Hidden, cl::desc("A flag that overrides the target's number of scalar registers.")); @@ -551,10 +535,7 @@ public: : NumPredStores(0), TheLoop(L), SE(SE), DL(DL), TLI(TLI), TheFunction(F), TTI(TTI), DT(DT), Induction(nullptr), WidestIndTy(nullptr), - LAI(F, L, SE, DL, TLI, AA, DT, - LoopAccessInfo::VectorizerParams( - MaxVectorWidth, VectorizationFactor, VectorizationInterleave, - RuntimeMemoryCheckThreshold)), + LAI(F, L, SE, DL, TLI, AA, DT), HasFunNoNaNAttr(false) {} /// This enum represents the kinds of reductions that we support. @@ -1021,7 +1002,7 @@ class LoopVectorizeHints { bool validate(unsigned Val) { switch (Kind) { case HK_WIDTH: - return isPowerOf2_32(Val) && Val <= MaxVectorWidth; + return isPowerOf2_32(Val) && Val <= VectorizerParams::MaxVectorWidth; case HK_UNROLL: return isPowerOf2_32(Val) && Val <= MaxInterleaveFactor; case HK_FORCE: @@ -1049,7 +1030,8 @@ public: }; LoopVectorizeHints(const Loop *L, bool DisableInterleaving) - : Width("vectorize.width", VectorizationFactor, HK_WIDTH), + : Width("vectorize.width", VectorizerParams::VectorizationFactor, + HK_WIDTH), Interleave("interleave.count", DisableInterleaving, HK_UNROLL), Force("vectorize.enable", FK_Undefined, HK_FORCE), TheLoop(L) { @@ -1057,8 +1039,8 @@ public: getHintsFromMetadata(); // force-vector-interleave overrides DisableInterleaving. - if (VectorizationInterleave.getNumOccurrences() > 0) - Interleave.Value = VectorizationInterleave; + if (VectorizerParams::isInterleaveForced()) + Interleave.Value = VectorizerParams::VectorizationInterleave; DEBUG(if (DisableInterleaving && Interleave.Value == 1) dbgs() << "LV: Interleaving disabled by the pass manager\n");