diff --git a/docs/LangRef.rst b/docs/LangRef.rst index bcf4f651829..649df4c5878 100644 --- a/docs/LangRef.rst +++ b/docs/LangRef.rst @@ -1749,6 +1749,45 @@ otherwise unsafe floating point operations dramatically change results in floating point (e.g. reassociate). This flag implies all the others. +.. _uselistorder: + +Use-list Order Directives +------------------------- + +Use-list directives encode the in-memory order of each use-list, allowing the +order to be recreated. ```` is a comma-separated list of +indexes that are assigned to the referenced value's uses. The referenced +value's use-list is immediately sorted by these indexes. + +Use-list directives may appear at function scope or global scope. They are not +instructions, and have no effect on the semantics of the IR. When they're at +function scope, they must appear after the terminator of the final basic block. + +If basic blocks have their address taken via ``blockaddress()`` expressions, +``uselistorder_bb`` can be used to reorder their use-lists from outside their +function's scope. + +:Syntax: + +:: + + uselistorder , { } + uselistorder_bb @function, %block { } + +:Examples: + +:: + + ; At function scope. + uselistorder i32 %arg1, { 1, 0, 2 } + uselistorder label %bb, { 1, 0 } + + ; At global scope. + uselistorder i32* @global, { 1, 2, 0 } + uselistorder i32 7, { 1, 0 } + uselistorder i32 (i32) @bar, { 1, 0 } + uselistorder_bb @foo, %bb, { 5, 1, 3, 2, 0, 4 } + .. _typesystem: Type System diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp index cb4a9557d43..e205a7abe4e 100644 --- a/lib/AsmParser/LLLexer.cpp +++ b/lib/AsmParser/LLLexer.cpp @@ -664,6 +664,10 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(x); KEYWORD(blockaddress); + // Use-list order directives. + KEYWORD(uselistorder); + KEYWORD(uselistorder_bb); + KEYWORD(personality); KEYWORD(cleanup); KEYWORD(catch); diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index 13b648ce9eb..177243cf647 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -239,6 +239,9 @@ bool LLParser::ParseTopLevelEntities() { } case lltok::kw_attributes: if (ParseUnnamedAttrGrp()) return true; break; + case lltok::kw_uselistorder: if (ParseUseListOrder()) return true; break; + case lltok::kw_uselistorder_bb: + if (ParseUseListOrderBB()) return true; break; } } } @@ -3371,8 +3374,7 @@ bool LLParser::PerFunctionState::resolveForwardRefBlockAddresses() { } /// ParseFunctionBody -/// ::= '{' BasicBlock+ '}' -/// +/// ::= '{' BasicBlock+ UseListOrderDirective* '}' bool LLParser::ParseFunctionBody(Function &Fn) { if (Lex.getKind() != lltok::lbrace) return TokError("expected '{' in function body"); @@ -3390,12 +3392,17 @@ bool LLParser::ParseFunctionBody(Function &Fn) { SaveAndRestore ScopeExit(BlockAddressPFS, &PFS); // We need at least one basic block. - if (Lex.getKind() == lltok::rbrace) + if (Lex.getKind() == lltok::rbrace || Lex.getKind() == lltok::kw_uselistorder) return TokError("function body requires at least one basic block"); - while (Lex.getKind() != lltok::rbrace) + while (Lex.getKind() != lltok::rbrace && + Lex.getKind() != lltok::kw_uselistorder) if (ParseBasicBlock(PFS)) return true; + while (Lex.getKind() != lltok::rbrace) + if (ParseUseListOrder(&PFS)) + return true; + // Eat the }. Lex.Lex(); @@ -4654,3 +4661,135 @@ bool LLParser::ParseMDNodeVector(SmallVectorImpl &Elts, return false; } + +//===----------------------------------------------------------------------===// +// Use-list order directives. +//===----------------------------------------------------------------------===// +bool LLParser::sortUseListOrder(Value *V, ArrayRef Indexes, + SMLoc Loc) { + if (V->use_empty()) + return Error(Loc, "value has no uses"); + + unsigned NumUses = 0; + SmallDenseMap Order; + for (const Use &U : V->uses()) { + if (++NumUses > Indexes.size()) + break; + Order[&U] = Indexes[NumUses - 1]; + } + if (NumUses < 2) + return Error(Loc, "value only has one use"); + if (Order.size() != Indexes.size() || NumUses > Indexes.size()) + return Error(Loc, "wrong number of indexes, expected " + + Twine(std::distance(V->use_begin(), V->use_end()))); + + V->sortUseList([&](const Use &L, const Use &R) { + return Order.lookup(&L) < Order.lookup(&R); + }); + return false; +} + +/// ParseUseListOrderIndexes +/// ::= '{' uint32 (',' uint32)+ '}' +bool LLParser::ParseUseListOrderIndexes(SmallVectorImpl &Indexes) { + SMLoc Loc = Lex.getLoc(); + if (ParseToken(lltok::lbrace, "expected '{' here")) + return true; + if (Lex.getKind() == lltok::rbrace) + return Lex.Error("expected non-empty list of uselistorder indexes"); + + // Use Offset, Max, and IsOrdered to check consistency of indexes. The + // indexes should be distinct numbers in the range [0, size-1], and should + // not be in order. + unsigned Offset = 0; + unsigned Max = 0; + bool IsOrdered = true; + assert(Indexes.empty() && "Expected empty order vector"); + do { + unsigned Index; + if (ParseUInt32(Index)) + return true; + + // Update consistency checks. + Offset += Index - Indexes.size(); + Max = std::max(Max, Index); + IsOrdered &= Index == Indexes.size(); + + Indexes.push_back(Index); + } while (EatIfPresent(lltok::comma)); + + if (ParseToken(lltok::rbrace, "expected '}' here")) + return true; + + if (Indexes.size() < 2) + return Error(Loc, "expected >= 2 uselistorder indexes"); + if (Offset != 0 || Max >= Indexes.size()) + return Error(Loc, "expected distinct uselistorder indexes in range [0, size)"); + if (IsOrdered) + return Error(Loc, "expected uselistorder indexes to change the order"); + + return false; +} + +/// ParseUseListOrder +/// ::= 'uselistorder' Type Value ',' UseListOrderIndexes +bool LLParser::ParseUseListOrder(PerFunctionState *PFS) { + SMLoc Loc = Lex.getLoc(); + if (ParseToken(lltok::kw_uselistorder, "expected uselistorder directive")) + return true; + + Value *V; + SmallVector Indexes; + if (ParseTypeAndValue(V, PFS) || + ParseToken(lltok::comma, "expected comma in uselistorder directive") || + ParseUseListOrderIndexes(Indexes)) + return true; + + return sortUseListOrder(V, Indexes, Loc); +} + +/// ParseUseListOrderBB +/// ::= 'uselistorder_bb' @foo ',' %bar ',' UseListOrderIndexes +bool LLParser::ParseUseListOrderBB() { + assert(Lex.getKind() == lltok::kw_uselistorder_bb); + SMLoc Loc = Lex.getLoc(); + Lex.Lex(); + + ValID Fn, Label; + SmallVector Indexes; + if (ParseValID(Fn) || + ParseToken(lltok::comma, "expected comma in uselistorder_bb directive") || + ParseValID(Label) || + ParseToken(lltok::comma, "expected comma in uselistorder_bb directive") || + ParseUseListOrderIndexes(Indexes)) + return true; + + // Check the function. + GlobalValue *GV; + if (Fn.Kind == ValID::t_GlobalName) + GV = M->getNamedValue(Fn.StrVal); + else if (Fn.Kind == ValID::t_GlobalID) + GV = Fn.UIntVal < NumberedVals.size() ? NumberedVals[Fn.UIntVal] : nullptr; + else + return Error(Fn.Loc, "expected function name in uselistorder_bb"); + if (!GV) + return Error(Fn.Loc, "invalid function forward reference in uselistorder_bb"); + auto *F = dyn_cast(GV); + if (!F) + return Error(Fn.Loc, "expected function name in uselistorder_bb"); + if (F->isDeclaration()) + return Error(Fn.Loc, "invalid declaration in uselistorder_bb"); + + // Check the basic block. + if (Label.Kind == ValID::t_LocalID) + return Error(Label.Loc, "invalid numeric label in uselistorder_bb"); + if (Label.Kind != ValID::t_LocalName) + return Error(Label.Loc, "expected basic block name in uselistorder_bb"); + Value *V = F->getValueSymbolTable().lookup(Label.StrVal); + if (!V) + return Error(Label.Loc, "invalid basic block in uselistorder_bb"); + if (!isa(V)) + return Error(Label.Loc, "expected basic block in uselistorder_bb"); + + return sortUseListOrder(V, Indexes, Loc); +} diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h index 556b385c3c4..3936c6e8731 100644 --- a/lib/AsmParser/LLParser.h +++ b/lib/AsmParser/LLParser.h @@ -438,6 +438,12 @@ namespace llvm { int ParseGetElementPtr(Instruction *&I, PerFunctionState &PFS); int ParseExtractValue(Instruction *&I, PerFunctionState &PFS); int ParseInsertValue(Instruction *&I, PerFunctionState &PFS); + + // Use-list order directives. + bool ParseUseListOrder(PerFunctionState *PFS = nullptr); + bool ParseUseListOrderBB(); + bool ParseUseListOrderIndexes(SmallVectorImpl &Indexes); + bool sortUseListOrder(Value *V, ArrayRef Indexes, SMLoc Loc); }; } // End llvm namespace diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h index 67cd6c9b755..2b39ebd2b57 100644 --- a/lib/AsmParser/LLToken.h +++ b/lib/AsmParser/LLToken.h @@ -180,6 +180,9 @@ namespace lltok { kw_extractelement, kw_insertelement, kw_shufflevector, kw_extractvalue, kw_insertvalue, kw_blockaddress, + // Use-list order directives. + kw_uselistorder, kw_uselistorder_bb, + // Unsigned Valued tokens (UIntVal). GlobalID, // @42 LocalVarID, // %42 diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index 7fcab5c921b..8aebc4396b6 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -49,6 +49,213 @@ AssemblyAnnotationWriter::~AssemblyAnnotationWriter() {} // Helper Functions //===----------------------------------------------------------------------===// +namespace { +struct OrderMap { + DenseMap> IDs; + + unsigned size() const { return IDs.size(); } + std::pair &operator[](const Value *V) { return IDs[V]; } + std::pair lookup(const Value *V) const { + return IDs.lookup(V); + } + void index(const Value *V) { + // Explicitly sequence get-size and insert-value operations to avoid UB. + unsigned ID = IDs.size() + 1; + IDs[V].first = ID; + } +}; +} + +static void orderValue(const Value *V, OrderMap &OM) { + if (OM.lookup(V).first) + return; + + if (const Constant *C = dyn_cast(V)) + if (C->getNumOperands() && !isa(C)) + for (const Value *Op : C->operands()) + if (!isa(Op) && !isa(Op)) + orderValue(Op, OM); + + // Note: we cannot cache this lookup above, since inserting into the map + // changes the map's size, and thus affects the other IDs. + OM.index(V); +} + +static OrderMap orderModule(const Module *M) { + // This needs to match the order used by ValueEnumerator::ValueEnumerator() + // and ValueEnumerator::incorporateFunction(). + OrderMap OM; + + for (const GlobalVariable &G : M->globals()) { + if (G.hasInitializer()) + if (!isa(G.getInitializer())) + orderValue(G.getInitializer(), OM); + orderValue(&G, OM); + } + for (const GlobalAlias &A : M->aliases()) { + if (!isa(A.getAliasee())) + orderValue(A.getAliasee(), OM); + orderValue(&A, OM); + } + for (const Function &F : *M) { + if (F.hasPrefixData()) + if (!isa(F.getPrefixData())) + orderValue(F.getPrefixData(), OM); + orderValue(&F, OM); + + if (F.isDeclaration()) + continue; + + for (const Argument &A : F.args()) + orderValue(&A, OM); + for (const BasicBlock &BB : F) { + orderValue(&BB, OM); + for (const Instruction &I : BB) { + for (const Value *Op : I.operands()) + if ((isa(*Op) && !isa(*Op)) || + isa(*Op)) + orderValue(Op, OM); + orderValue(&I, OM); + } + } + } + return OM; +} + +static void predictValueUseListOrderImpl(const Value *V, const Function *F, + unsigned ID, const OrderMap &OM, + UseListOrderStack &Stack) { + // Predict use-list order for this one. + typedef std::pair Entry; + SmallVector List; + for (const Use &U : V->uses()) + // Check if this user will be serialized. + if (OM.lookup(U.getUser()).first) + List.push_back(std::make_pair(&U, List.size())); + + if (List.size() < 2) + // We may have lost some users. + return; + + bool GetsReversed = + !isa(V) && !isa(V) && !isa(V); + if (auto *BA = dyn_cast(V)) + ID = OM.lookup(BA->getBasicBlock()).first; + std::sort(List.begin(), List.end(), [&](const Entry &L, const Entry &R) { + const Use *LU = L.first; + const Use *RU = R.first; + if (LU == RU) + return false; + + auto LID = OM.lookup(LU->getUser()).first; + auto RID = OM.lookup(RU->getUser()).first; + + // If ID is 4, then expect: 7 6 5 1 2 3. + if (LID < RID) { + if (GetsReversed) + if (RID <= ID) + return true; + return false; + } + if (RID < LID) { + if (GetsReversed) + if (LID <= ID) + return false; + return true; + } + + // LID and RID are equal, so we have different operands of the same user. + // Assume operands are added in order for all instructions. + if (GetsReversed) + if (LID <= ID) + return LU->getOperandNo() < RU->getOperandNo(); + return LU->getOperandNo() > RU->getOperandNo(); + }); + + if (std::is_sorted( + List.begin(), List.end(), + [](const Entry &L, const Entry &R) { return L.second < R.second; })) + // Order is already correct. + return; + + // Store the shuffle. + Stack.emplace_back(V, F, List.size()); + assert(List.size() == Stack.back().Shuffle.size() && "Wrong size"); + for (size_t I = 0, E = List.size(); I != E; ++I) + Stack.back().Shuffle[I] = List[I].second; +} + +static void predictValueUseListOrder(const Value *V, const Function *F, + OrderMap &OM, UseListOrderStack &Stack) { + auto &IDPair = OM[V]; + assert(IDPair.first && "Unmapped value"); + if (IDPair.second) + // Already predicted. + return; + + // Do the actual prediction. + IDPair.second = true; + if (!V->use_empty() && std::next(V->use_begin()) != V->use_end()) + predictValueUseListOrderImpl(V, F, IDPair.first, OM, Stack); + + // Recursive descent into constants. + if (const Constant *C = dyn_cast(V)) + if (C->getNumOperands()) // Visit GlobalValues. + for (const Value *Op : C->operands()) + if (isa(Op)) // Visit GlobalValues. + predictValueUseListOrder(Op, F, OM, Stack); +} + +static UseListOrderStack predictUseListOrder(const Module *M) { + OrderMap OM = orderModule(M); + + // Use-list orders need to be serialized after all the users have been added + // to a value, or else the shuffles will be incomplete. Store them per + // function in a stack. + // + // Aside from function order, the order of values doesn't matter much here. + UseListOrderStack Stack; + + // We want to visit the functions backward now so we can list function-local + // constants in the last Function they're used in. Module-level constants + // have already been visited above. + for (auto I = M->rbegin(), E = M->rend(); I != E; ++I) { + const Function &F = *I; + if (F.isDeclaration()) + continue; + for (const BasicBlock &BB : F) + predictValueUseListOrder(&BB, &F, OM, Stack); + for (const Argument &A : F.args()) + predictValueUseListOrder(&A, &F, OM, Stack); + for (const BasicBlock &BB : F) + for (const Instruction &I : BB) + for (const Value *Op : I.operands()) + if (isa(*Op) || isa(*Op)) // Visit GlobalValues. + predictValueUseListOrder(Op, &F, OM, Stack); + for (const BasicBlock &BB : F) + for (const Instruction &I : BB) + predictValueUseListOrder(&I, &F, OM, Stack); + } + + // Visit globals last. + for (const GlobalVariable &G : M->globals()) + predictValueUseListOrder(&G, nullptr, OM, Stack); + for (const Function &F : *M) + predictValueUseListOrder(&F, nullptr, OM, Stack); + for (const GlobalAlias &A : M->aliases()) + predictValueUseListOrder(&A, nullptr, OM, Stack); + for (const GlobalVariable &G : M->globals()) + if (G.hasInitializer()) + predictValueUseListOrder(G.getInitializer(), nullptr, OM, Stack); + for (const GlobalAlias &A : M->aliases()) + predictValueUseListOrder(A.getAliasee(), nullptr, OM, Stack); + for (const Function &F : *M) + if (F.hasPrefixData()) + predictValueUseListOrder(F.getPrefixData(), nullptr, OM, Stack); + + return Stack; +} + static const Module *getModuleFromVal(const Value *V) { if (const Argument *MA = dyn_cast(V)) return MA->getParent() ? MA->getParent()->getParent() : nullptr; @@ -347,6 +554,8 @@ public: FunctionProcessed = false; } + const Function *getFunction() const { return TheFunction; } + /// After calling incorporateFunction, use this method to remove the /// most recently incorporated function from the SlotTracker. This /// will reset the state of the machine back to just the module contents. @@ -1279,6 +1488,9 @@ void AssemblyWriter::writeParamOperand(const Value *Operand, void AssemblyWriter::printModule(const Module *M) { Machine.initialize(); + if (shouldPreserveAssemblyUseListOrder()) + UseListOrders = predictUseListOrder(M); + if (!M->getModuleIdentifier().empty() && // Don't print the ID if it will start a new line (which would // require a comment char before it). @@ -1339,9 +1551,13 @@ void AssemblyWriter::printModule(const Module *M) { I != E; ++I) printAlias(I); + // Output global use-lists. + printUseLists(nullptr); + // Output all of the functions. for (Module::const_iterator I = M->begin(), E = M->end(); I != E; ++I) printFunction(I); + assert(UseListOrders.empty() && "All use-lists should have been consumed"); // Output all attribute groups. if (!Machine.as_empty()) { @@ -1692,6 +1908,9 @@ void AssemblyWriter::printFunction(const Function *F) { for (Function::const_iterator I = F->begin(), E = F->end(); I != E; ++I) printBasicBlock(I); + // Output the function's use-lists. + printUseLists(F); + Out << "}\n"; } @@ -2169,6 +2388,45 @@ void AssemblyWriter::writeAllAttributeGroups() { } // namespace llvm +void AssemblyWriter::printUseListOrder(const UseListOrder &Order) { + bool IsInFunction = Machine.getFunction(); + if (IsInFunction) + Out << " "; + + Out << "uselistorder"; + if (const BasicBlock *BB = + IsInFunction ? nullptr : dyn_cast(Order.V)) { + Out << "_bb "; + writeOperand(BB->getParent(), false); + Out << ", "; + writeOperand(BB, false); + } else { + Out << " "; + writeOperand(Order.V, true); + } + Out << ", { "; + + assert(Order.Shuffle.size() >= 2 && "Shuffle too small"); + Out << Order.Shuffle[0]; + for (unsigned I = 1, E = Order.Shuffle.size(); I != E; ++I) + Out << ", " << Order.Shuffle[I]; + Out << " }\n"; +} + +void AssemblyWriter::printUseLists(const Function *F) { + auto hasMore = + [&]() { return !UseListOrders.empty() && UseListOrders.back().F == F; }; + if (!hasMore()) + // Nothing to do. + return; + + Out << "\n; uselistorder directives\n"; + while (hasMore()) { + printUseListOrder(UseListOrders.back()); + UseListOrders.pop_back(); + } +} + //===----------------------------------------------------------------------===// // External Interface declarations //===----------------------------------------------------------------------===// diff --git a/lib/IR/AsmWriter.h b/lib/IR/AsmWriter.h index cebfca53e74..60da5ad3350 100644 --- a/lib/IR/AsmWriter.h +++ b/lib/IR/AsmWriter.h @@ -20,6 +20,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/TypeFinder.h" +#include "llvm/IR/UseListOrder.h" #include "llvm/Support/FormattedStream.h" namespace llvm { @@ -73,6 +74,7 @@ private: TypePrinting TypePrinter; AssemblyAnnotationWriter *AnnotationWriter; SetVector Comdats; + UseListOrderStack UseListOrders; public: /// Construct an AssemblyWriter with an external SlotTracker @@ -111,6 +113,9 @@ public: void printInstructionLine(const Instruction &I); void printInstruction(const Instruction &I); + void printUseListOrder(const UseListOrder &Order); + void printUseLists(const Function *F); + private: void init(); diff --git a/test/Assembler/alias-use-list-order.ll b/test/Assembler/alias-use-list-order.ll new file mode 100644 index 00000000000..d29fd6e435f --- /dev/null +++ b/test/Assembler/alias-use-list-order.ll @@ -0,0 +1,11 @@ +; RUN: verify-uselistorder < %s + +; Globals. +@global = global i32 0 +@alias.ref1 = global i32* getelementptr inbounds (i32* @alias, i64 1) +@alias.ref2 = global i32* getelementptr inbounds (i32* @alias, i64 1) + +; Aliases. +@alias = alias i32* @global +@alias.ref3 = alias i32* getelementptr inbounds (i32* @alias, i64 1) +@alias.ref4 = alias i32* getelementptr inbounds (i32* @alias, i64 1) diff --git a/test/Assembler/invalid-uselistorder-function-between-blocks.ll b/test/Assembler/invalid-uselistorder-function-between-blocks.ll new file mode 100644 index 00000000000..8f771e86dd4 --- /dev/null +++ b/test/Assembler/invalid-uselistorder-function-between-blocks.ll @@ -0,0 +1,37 @@ +; RUN: not llvm-as < %s -disable-output 2>&1 | FileCheck %s +; CHECK: error: expected uselistorder directive + +define i32 @f32(i32 %a, i32 %b, i32 %c, i32 %d) { +entry: + br label %first + +;