From 048f520b91e5fcabb1aa85cae3ec522ab5a539e6 Mon Sep 17 00:00:00 2001 From: "Duncan P. N. Exon Smith" Date: Tue, 29 Apr 2014 16:15:35 +0000 Subject: [PATCH] Support: Add BranchProbability::scale() and ::scaleByInverse() Add API to `BranchProbability` for scaling big integers. Next job is to rip the logic out of `BlockMass` and `BlockFrequency`. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@207544 91177308-0d34-0410-b5e6-96231b3b80d8 --- include/llvm/Support/BranchProbability.h | 24 ++++++++ lib/Support/BranchProbability.cpp | 47 ++++++++++++++ unittests/Support/BranchProbabilityTest.cpp | 68 +++++++++++++++++++++ 3 files changed, 139 insertions(+) diff --git a/include/llvm/Support/BranchProbability.h b/include/llvm/Support/BranchProbability.h index bf6b01defad..74d64194cf6 100644 --- a/include/llvm/Support/BranchProbability.h +++ b/include/llvm/Support/BranchProbability.h @@ -50,6 +50,30 @@ public: void dump() const; + /// \brief Scale a large integer. + /// + /// Scales \c Num. Guarantees full precision. Returns the floor of the + /// result. + /// + /// \return \c Num times \c this. + /// + /// \note This code should be shared with (or replaced by) the implementation + /// of \a BlockFrequency::scale(), which seems to be calculating something + /// similar. + uint64_t scale(uint64_t Num) const; + + /// \brief Scale a large integer by the inverse. + /// + /// Scales \c Num by the inverse of \c this. Guarantees full precision. + /// Returns the floor of the result. + /// + /// \return \c Num divided by \c this. + /// + /// \note This code should be shared with (or replaced by) the implementation + /// of \a BlockFrequency::scale(), which seems to be calculating something + /// similar. + uint64_t scaleByInverse(uint64_t Num) const; + bool operator==(BranchProbability RHS) const { return (uint64_t)N * RHS.D == (uint64_t)D * RHS.N; } diff --git a/lib/Support/BranchProbability.cpp b/lib/Support/BranchProbability.cpp index e8b83e59802..c541fca3fb4 100644 --- a/lib/Support/BranchProbability.cpp +++ b/lib/Support/BranchProbability.cpp @@ -26,6 +26,53 @@ void BranchProbability::dump() const { dbgs() << *this << '\n'; } +static uint64_t scale(uint64_t Num, uint32_t N, uint32_t D) { + assert(D && "divide by 0"); + + // Fast path for multiplying by 1.0. + if (!Num || D == N) + return Num; + + // Split Num into upper and lower parts to multiply, then recombine. + uint64_t ProductHigh = (Num >> 32) * N; + uint64_t ProductLow = (Num & UINT32_MAX) * N; + + // Split into 32-bit digits. + uint32_t Upper32 = ProductHigh >> 32; + uint32_t Lower32 = ProductLow & UINT32_MAX; + uint32_t Mid32Partial = ProductHigh & UINT32_MAX; + uint32_t Mid32 = Mid32Partial + (ProductLow >> 32); + + // Carry. + Upper32 += Mid32 < Mid32Partial; + + // Check for overflow. + if (Upper32 >= D) + return UINT64_MAX; + + uint64_t Rem = (uint64_t(Upper32) << 32) | Mid32; + uint64_t UpperQ = Rem / D; + + // Check for overflow. + if (UpperQ > UINT32_MAX) + return UINT64_MAX; + + Rem = ((Rem % D) << 32) | Lower32; + uint64_t LowerQ = Rem / D; + uint64_t Q = (UpperQ << 32) + LowerQ; + + // Check for overflow. + return Q < LowerQ ? UINT64_MAX : Q; +} + +uint64_t BranchProbability::scale(uint64_t Num) const { + return ::scale(Num, N, D); +} + +uint64_t BranchProbability::scaleByInverse(uint64_t Num) const { + return ::scale(Num, D, N); +} + namespace llvm { raw_ostream &operator<<(raw_ostream &OS, const BranchProbability &Prob) { diff --git a/unittests/Support/BranchProbabilityTest.cpp b/unittests/Support/BranchProbabilityTest.cpp index 56ab22334ca..b528728e8ae 100644 --- a/unittests/Support/BranchProbabilityTest.cpp +++ b/unittests/Support/BranchProbabilityTest.cpp @@ -87,4 +87,72 @@ TEST(BranchProbabilityTest, getCompl) { EXPECT_EQ(BP::getOne(), BP(0, 7).getCompl()); } +TEST(BranchProbabilityTest, scale) { + // Multiply by 1.0. + EXPECT_EQ(UINT64_MAX, BP(1, 1).scale(UINT64_MAX)); + EXPECT_EQ(UINT64_MAX, BP(7, 7).scale(UINT64_MAX)); + EXPECT_EQ(UINT32_MAX, BP(1, 1).scale(UINT32_MAX)); + EXPECT_EQ(UINT32_MAX, BP(7, 7).scale(UINT32_MAX)); + EXPECT_EQ(0u, BP(1, 1).scale(0)); + EXPECT_EQ(0u, BP(7, 7).scale(0)); + + // Multiply by 0.0. + EXPECT_EQ(0u, BP(0, 1).scale(UINT64_MAX)); + EXPECT_EQ(0u, BP(0, 1).scale(UINT64_MAX)); + EXPECT_EQ(0u, BP(0, 1).scale(0)); + + auto Two63 = UINT64_C(1) << 63; + auto Two31 = UINT64_C(1) << 31; + + // Multiply by 0.5. + EXPECT_EQ(Two63 - 1, BP(1, 2).scale(UINT64_MAX)); + + // Big fractions. + EXPECT_EQ(1u, BP(Two31, UINT32_MAX).scale(2)); + EXPECT_EQ(Two31, BP(Two31, UINT32_MAX).scale(Two31 * 2)); + EXPECT_EQ(Two63 + Two31, BP(Two31, UINT32_MAX).scale(UINT64_MAX)); + + // High precision. + EXPECT_EQ(UINT64_C(9223372047592194055), + BP(Two31 + 1, UINT32_MAX - 2).scale(UINT64_MAX)); +} + +TEST(BranchProbabilityTest, scaleByInverse) { + // Divide by 1.0. + EXPECT_EQ(UINT64_MAX, BP(1, 1).scaleByInverse(UINT64_MAX)); + EXPECT_EQ(UINT64_MAX, BP(7, 7).scaleByInverse(UINT64_MAX)); + EXPECT_EQ(UINT32_MAX, BP(1, 1).scaleByInverse(UINT32_MAX)); + EXPECT_EQ(UINT32_MAX, BP(7, 7).scaleByInverse(UINT32_MAX)); + EXPECT_EQ(0u, BP(1, 1).scaleByInverse(0)); + EXPECT_EQ(0u, BP(7, 7).scaleByInverse(0)); + + // Divide by something very small. + EXPECT_EQ(UINT64_MAX, BP(1, UINT32_MAX).scaleByInverse(UINT64_MAX)); + EXPECT_EQ(uint64_t(UINT32_MAX) * UINT32_MAX, + BP(1, UINT32_MAX).scaleByInverse(UINT32_MAX)); + EXPECT_EQ(UINT32_MAX, BP(1, UINT32_MAX).scaleByInverse(1)); + + auto Two63 = UINT64_C(1) << 63; + auto Two31 = UINT64_C(1) << 31; + + // Divide by 0.5. + EXPECT_EQ(UINT64_MAX - 1, BP(1, 2).scaleByInverse(Two63 - 1)); + EXPECT_EQ(UINT64_MAX, BP(1, 2).scaleByInverse(Two63)); + + // Big fractions. + EXPECT_EQ(1u, BP(Two31, UINT32_MAX).scaleByInverse(1)); + EXPECT_EQ(2u, BP(Two31 - 1, UINT32_MAX).scaleByInverse(1)); + EXPECT_EQ(Two31 * 2 - 1, BP(Two31, UINT32_MAX).scaleByInverse(Two31)); + EXPECT_EQ(Two31 * 2 + 1, BP(Two31 - 1, UINT32_MAX).scaleByInverse(Two31)); + EXPECT_EQ(UINT64_MAX, BP(Two31, UINT32_MAX).scaleByInverse(Two63 + Two31)); + + // High precision. The exact answers to these are close to the successors of + // the floor. If we were rounding, these would round up. + EXPECT_EQ(UINT64_C(18446744065119617030), + BP(Two31 + 2, UINT32_MAX - 2) + .scaleByInverse(UINT64_C(9223372047592194055))); + EXPECT_EQ(UINT64_C(18446744065119617026), + BP(Two31 + 1, UINT32_MAX).scaleByInverse(Two63 + Two31)); +} + }