AArch64: remove ConstantIsland pass & put literals in separate section.

This implements the review suggestion to simplify the AArch64 backend. If we
later discover that we *really* need the extra complexity of the
ConstantIslands pass for performance reasons it can be resurrected.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@175258 91177308-0d34-0410-b5e6-96231b3b80d8

lib/Target/AArch64/AArch64.h

@@ -29,8 +29,6 @@ class MCInst;
 FunctionPass *createAArch64ISelDAG(AArch64TargetMachine &TM,
                                    CodeGenOpt::Level OptLevel);
 
-FunctionPass *createAArch64ConstantIslandPass();
-
 FunctionPass *createAArch64CleanupLocalDynamicTLSPass();
 
 void LowerAArch64MachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,

lib/Target/AArch64/AArch64AsmPrinter.cpp

@@ -17,7 +17,6 @@
 #include "InstPrinter/AArch64InstPrinter.h"
 #include "llvm/DebugInfo.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -298,20 +297,6 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
 
   switch (MI->getOpcode()) {
-  case AArch64::CONSTPOOL_ENTRY: {
-    unsigned LabelId = (unsigned)MI->getOperand(0).getImm();
-    unsigned CPIdx = (unsigned)MI->getOperand(1).getIndex();
-
-    OutStreamer.EmitLabel(GetCPISymbol(LabelId));
-
-    const MachineConstantPoolEntry &MCPE = MCP->getConstants()[CPIdx];
-    if (MCPE.isMachineConstantPoolEntry())
-      EmitMachineConstantPoolValue(MCPE.Val.MachineCPVal);
-    else
-      EmitGlobalConstant(MCPE.Val.ConstVal);
-
-    return;
-  }
   case AArch64::DBG_VALUE: {
     if (isVerbose() && OutStreamer.hasRawTextSupport()) {
       SmallString<128> TmpStr;
@@ -352,7 +337,6 @@ void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
 }
 
 bool AArch64AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
-  MCP = MF.getConstantPool();
   return AsmPrinter::runOnMachineFunction(MF);
 }
 

lib/Target/AArch64/AArch64AsmPrinter.h

@@ -29,7 +29,6 @@ class LLVM_LIBRARY_VISIBILITY AArch64AsmPrinter : public AsmPrinter {
   /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
   /// make the right decision when printing asm code for different targets.
   const AArch64Subtarget *Subtarget;
-  const MachineConstantPool *MCP;
 
   // emitPseudoExpansionLowering - tblgen'erated.
   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
@@ -74,10 +73,6 @@ class LLVM_LIBRARY_VISIBILITY AArch64AsmPrinter : public AsmPrinter {
     return "AArch64 Assembly Printer";
   }
 
-  /// A no-op on AArch64 because we emit our constant pool entries inline with
-  /// the function.
-  virtual void EmitConstantPool() {}
-
   virtual bool runOnMachineFunction(MachineFunction &MF);
 };
 } // end namespace llvm

lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

@@ -89,8 +89,8 @@ public:
   bool SelectTSTBOperand(SDValue N, SDValue &FixedPos, unsigned RegWidth);
 
   SDNode *TrySelectToMoveImm(SDNode *N);
+  SDNode *LowerToFPLitPool(SDNode *Node);
   SDNode *SelectToLitPool(SDNode *N);
-  SDNode *SelectToFPLitPool(SDNode *N);
 
   SDNode* Select(SDNode*);
 private:
@@ -225,92 +225,78 @@ SDNode *AArch64DAGToDAGISel::TrySelectToMoveImm(SDNode *Node) {
 }
 
 SDNode *AArch64DAGToDAGISel::SelectToLitPool(SDNode *Node) {
-  DebugLoc dl = Node->getDebugLoc();
+  DebugLoc DL = Node->getDebugLoc();
   uint64_t UnsignedVal = cast<ConstantSDNode>(Node)->getZExtValue();
   int64_t SignedVal = cast<ConstantSDNode>(Node)->getSExtValue();
   EVT DestType = Node->getValueType(0);
+  EVT PtrVT = TLI.getPointerTy();
 
   // Since we may end up loading a 64-bit constant from a 32-bit entry the
   // constant in the pool may have a different type to the eventual node.
-  SDValue PoolEntry;
+  ISD::LoadExtType Extension;
-  EVT LoadType;
+  EVT MemType;
-  unsigned LoadInst;
 
   assert((DestType == MVT::i64 || DestType == MVT::i32)
          && "Only expect integer constants at the moment");
 
-  if (DestType == MVT::i32 || UnsignedVal <= UINT32_MAX) {
+  if (DestType == MVT::i32) {
-    // LDR w3, lbl
+    Extension = ISD::NON_EXTLOAD;
-    LoadInst = AArch64::LDRw_lit;
+    MemType = MVT::i32;
-    LoadType = MVT::i32;
+  } else if (UnsignedVal <= UINT32_MAX) {
-
+    Extension = ISD::ZEXTLOAD;
-    PoolEntry = CurDAG->getTargetConstantPool(
+    MemType = MVT::i32;
-      ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), UnsignedVal),
-      MVT::i32);
   } else if (SignedVal >= INT32_MIN && SignedVal <= INT32_MAX) {
-    // We can use a sign-extending 32-bit load: LDRSW x3, lbl
+    Extension = ISD::SEXTLOAD;
-    LoadInst = AArch64::LDRSWx_lit;
+    MemType = MVT::i32;
-    LoadType = MVT::i64;
-
-    PoolEntry = CurDAG->getTargetConstantPool(
-      ConstantInt::getSigned(Type::getInt32Ty(*CurDAG->getContext()),
-                             SignedVal),
-      MVT::i32);
   } else {
-    // Full 64-bit load needed: LDR x3, lbl
+    Extension = ISD::NON_EXTLOAD;
-    LoadInst = AArch64::LDRx_lit;
+    MemType = MVT::i64;
-    LoadType = MVT::i64;
-
-    PoolEntry = CurDAG->getTargetConstantPool(
-      ConstantInt::get(Type::getInt64Ty(*CurDAG->getContext()), UnsignedVal),
-      MVT::i64);
   }
 
-  SDNode *ResNode = CurDAG->getMachineNode(LoadInst, dl,
+  Constant *CV = ConstantInt::get(Type::getIntNTy(*CurDAG->getContext(),
-                                           LoadType, MVT::Other,
+                                                  MemType.getSizeInBits()),
-                                           PoolEntry, CurDAG->getEntryNode());
+                                  UnsignedVal);
+  SDValue PoolAddr;
+  unsigned Alignment = TLI.getDataLayout()->getABITypeAlignment(CV->getType());
+  PoolAddr = CurDAG->getNode(AArch64ISD::WrapperSmall, DL, PtrVT,
+                             CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0,
+                                                         AArch64II::MO_NO_FLAG),
+                             CurDAG->getTargetConstantPool(CV, PtrVT, 0, 0,
+                                                           AArch64II::MO_LO12),
+                             CurDAG->getConstant(Alignment, MVT::i32));
 
-  if (DestType != LoadType) {
+  return CurDAG->getExtLoad(Extension, DL, DestType, CurDAG->getEntryNode(),
-    // We used the implicit zero-extension of "LDR w3, lbl", tell LLVM this
+                            PoolAddr,
-    // fact.
+                            MachinePointerInfo::getConstantPool(), MemType,
-    assert(DestType == MVT::i64 && LoadType == MVT::i32
+                            /* isVolatile = */ false,
-           && "Unexpected load combination");
+                            /* isNonTemporal = */ false,
-
+                            Alignment).getNode();
-    ResNode = CurDAG->getMachineNode(TargetOpcode::SUBREG_TO_REG, dl,
-                          MVT::i64, MVT::i32, MVT::Other,
-                          CurDAG->getTargetConstant(0, MVT::i64),
-                          SDValue(ResNode, 0),
-                          CurDAG->getTargetConstant(AArch64::sub_32, MVT::i32));
-  }
-
-  return ResNode;
 }
 
-SDNode *AArch64DAGToDAGISel::SelectToFPLitPool(SDNode *Node) {
+SDNode *AArch64DAGToDAGISel::LowerToFPLitPool(SDNode *Node) {
-  DebugLoc dl = Node->getDebugLoc();
+  DebugLoc DL = Node->getDebugLoc();
   const ConstantFP *FV = cast<ConstantFPSDNode>(Node)->getConstantFPValue();
+  EVT PtrVT = TLI.getPointerTy();
   EVT DestType = Node->getValueType(0);
 
-  unsigned LoadInst;
+  unsigned Alignment = TLI.getDataLayout()->getABITypeAlignment(FV->getType());
-  switch (DestType.getSizeInBits()) {
+  SDValue PoolAddr;
-  case 32:
-      LoadInst = AArch64::LDRs_lit;
-      break;
-  case 64:
-      LoadInst = AArch64::LDRd_lit;
-      break;
-  case 128:
-      LoadInst = AArch64::LDRq_lit;
-      break;
-  default: llvm_unreachable("cannot select floating-point litpool");
-  }
 
-  SDValue PoolEntry = CurDAG->getTargetConstantPool(FV, DestType);
+  assert(TM.getCodeModel() == CodeModel::Small &&
-  SDNode *ResNode = CurDAG->getMachineNode(LoadInst, dl,
+         "Only small code model supported");
-                                           DestType, MVT::Other,
+  PoolAddr = CurDAG->getNode(AArch64ISD::WrapperSmall, DL, PtrVT,
-                                           PoolEntry, CurDAG->getEntryNode());
+                             CurDAG->getTargetConstantPool(FV, PtrVT, 0, 0,
+                                                         AArch64II::MO_NO_FLAG),
+                             CurDAG->getTargetConstantPool(FV, PtrVT, 0, 0,
+                                                           AArch64II::MO_LO12),
+                             CurDAG->getConstant(Alignment, MVT::i32));
 
-  return ResNode;
+  return CurDAG->getLoad(DestType, DL, CurDAG->getEntryNode(), PoolAddr,
+                         MachinePointerInfo::getConstantPool(),
+                         /* isVolatile = */ false,
+                         /* isNonTemporal = */ false,
+                         /* isInvariant = */ true,
+                         Alignment).getNode();
 }
 
 bool
@@ -377,17 +363,19 @@ SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
       ResNode = TrySelectToMoveImm(Node);
     }
 
-    // If even that fails we fall back to a lit-pool entry at the moment. Future
+    if (ResNode)
-    // tuning or restrictions like non-readable code-sections may mandate a
+      return ResNode;
-    // sequence of MOVZ/MOVN/MOVK instructions.
-    if (!ResNode) {
-      ResNode = SelectToLitPool(Node);
-    }
 
+    // If even that fails we fall back to a lit-pool entry at the moment. Future
+    // tuning may change this to a sequence of MOVZ/MOVN/MOVK instructions.
+    ResNode = SelectToLitPool(Node);
     assert(ResNode && "We need *some* way to materialise a constant");
 
+    // We want to continue selection at this point since the litpool access
+    // generated used generic nodes for simplicity.
     ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0));
-    return NULL;
+    Node = ResNode;
+    break;
   }
   case ISD::ConstantFP: {
     if (A64Imms::isFPImm(cast<ConstantFPSDNode>(Node)->getValueAPF())) {
@@ -395,9 +383,13 @@ SDNode *AArch64DAGToDAGISel::Select(SDNode *Node) {
       break;
     }
 
-    SDNode *ResNode = SelectToFPLitPool(Node);
+    SDNode *ResNode = LowerToFPLitPool(Node);
     ReplaceUses(SDValue(Node, 0), SDValue(ResNode, 0));
-    return NULL;
+
+    // We want to continue selection at this point since the litpool access
+    // generated used generic nodes for simplicity.
+    Node = ResNode;
+    break;
   }
   default:
     break; // Let generic code handle it

lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -1866,8 +1866,14 @@ AArch64TargetLowering::LowerGlobalAddressELF(SDValue Op,
     // Weak symbols can't use ADRP/ADD pair since they should evaluate to
     // zero when undefined. In PIC mode the GOT can take care of this, but in
     // absolute mode we use a constant pool load.
-    return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
+    SDValue PoolAddr;
-                       DAG.getConstantPool(GV, GN->getValueType(0)),
+    PoolAddr = DAG.getNode(AArch64ISD::WrapperSmall, dl, PtrVT,
+                           DAG.getTargetConstantPool(GV, PtrVT, 0, 0,
+                                                     AArch64II::MO_NO_FLAG),
+                           DAG.getTargetConstantPool(GV, PtrVT, 0, 0,
+                                                     AArch64II::MO_LO12),
+                           DAG.getConstant(8, MVT::i32));
+    return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), PoolAddr,
                        MachinePointerInfo::getConstantPool(),
                        /*isVolatile=*/ false,  /*isNonTemporal=*/ true,
                        /*isInvariant=*/ true, 8);

lib/Target/AArch64/AArch64InstrInfo.cpp

@@ -558,50 +558,6 @@ void AArch64InstrInfo::getAddressConstraints(const MachineInstr &MI,
   }
 }
 
-unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
-  const MCInstrDesc &MCID = MI.getDesc();
-  const MachineBasicBlock &MBB = *MI.getParent();
-  const MachineFunction &MF = *MBB.getParent();
-  const MCAsmInfo &MAI = *MF.getTarget().getMCAsmInfo();
-
-  if (MCID.getSize())
-    return MCID.getSize();
-
-  if (MI.getOpcode() == AArch64::INLINEASM)
-    return getInlineAsmLength(MI.getOperand(0).getSymbolName(), MAI);
-
-  if (MI.isLabel())
-    return 0;
-
-  switch (MI.getOpcode()) {
-  case TargetOpcode::BUNDLE:
-    return getInstBundleLength(MI);
-  case TargetOpcode::IMPLICIT_DEF:
-  case TargetOpcode::KILL:
-  case TargetOpcode::PROLOG_LABEL:
-  case TargetOpcode::EH_LABEL:
-  case TargetOpcode::DBG_VALUE:
-    return 0;
-  case AArch64::CONSTPOOL_ENTRY:
-    return MI.getOperand(2).getImm();
-  case AArch64::TLSDESCCALL:
-    return 0;
-  default:
-    llvm_unreachable("Unknown instruction class");
-  }
-}
-
-unsigned AArch64InstrInfo::getInstBundleLength(const MachineInstr &MI) const {
-  unsigned Size = 0;
-  MachineBasicBlock::const_instr_iterator I = MI;
-  MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
-  while (++I != E && I->isInsideBundle()) {
-    assert(!I->isBundle() && "No nested bundle!");
-    Size += getInstSizeInBytes(*I);
-  }
-  return Size;
-}
-
 bool llvm::rewriteA64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
                                 unsigned FrameReg, int &Offset,
                                 const AArch64InstrInfo &TII) {

lib/Target/AArch64/AArch64InstrInfo.h

@@ -83,10 +83,6 @@ public:
   ///    + imm % OffsetScale == 0
   void getAddressConstraints(const MachineInstr &MI, int &AccessScale,
                              int &MinOffset, int &MaxOffset) const;
-
-  unsigned getInstSizeInBytes(const MachineInstr &MI) const;
-
-  unsigned getInstBundleLength(const MachineInstr &MI) const;
 };
 
 bool rewriteA64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,

lib/Target/AArch64/AArch64InstrInfo.td

@@ -2449,16 +2449,12 @@ class A64I_LDRlitSimple<bits<2> opc, bit v, RegisterClass OutReg,
                  "ldr\t$Rt, $Imm19", patterns, NoItinerary>;
 
 let mayLoad = 1 in {
-  def LDRw_lit : A64I_LDRlitSimple<0b00, 0b0, GPR32,
+  def LDRw_lit : A64I_LDRlitSimple<0b00, 0b0, GPR32>;
-                              [(set (i32 GPR32:$Rt), (load constpool:$Imm19))]>;
+  def LDRx_lit : A64I_LDRlitSimple<0b01, 0b0, GPR64>;
-  def LDRx_lit : A64I_LDRlitSimple<0b01, 0b0, GPR64,
-                              [(set (i64 GPR64:$Rt), (load constpool:$Imm19))]>;
 }
 
-def LDRs_lit  : A64I_LDRlitSimple<0b00, 0b1, FPR32,
+def LDRs_lit  : A64I_LDRlitSimple<0b00, 0b1, FPR32>;
-                              [(set (f32 FPR32:$Rt), (load constpool:$Imm19))]>;
+def LDRd_lit  : A64I_LDRlitSimple<0b01, 0b1, FPR64>;
-def LDRd_lit  : A64I_LDRlitSimple<0b01, 0b1, FPR64,
-                              [(set (f64 FPR64:$Rt), (load constpool:$Imm19))]>;
 
 let mayLoad = 1 in {
   def LDRq_lit : A64I_LDRlitSimple<0b10, 0b1, FPR128>;
@@ -4564,22 +4560,6 @@ def : Pat<(and (A64Bfi GPR64:$src, GPR64:$Rn, imm:$ImmR, imm:$ImmS),
 
 }
 
-//===----------------------------------------------------------------------===//
-// Constant island entries
-//===----------------------------------------------------------------------===//
-
-// The constant island pass needs to create "instructions" in the middle of the
-// instruction stream to reresent its constants.
-
-def cpinst_operand : Operand<i32>;
-
-def CONSTPOOL_ENTRY : PseudoInst<(outs), (ins cpinst_operand:$instid,
-                                              cpinst_operand:$cpidx,
-                                              i32imm:$size), []> {
-  let hasSideEffects = 0;
-  let isNotDuplicable = 1;
-}
-
 //===----------------------------------------------------------------------===//
 // Miscellaneous patterns
 //===----------------------------------------------------------------------===//
@@ -4898,6 +4878,8 @@ defm : uimm12_pats<(A64WrapperSmall texternalsym:$Hi, texternalsym:$Lo12,
                                     ALIGN),
                    (ADRPxi texternalsym:$Hi), (i64 texternalsym:$Lo12)>;
 
+defm : uimm12_pats<(A64WrapperSmall tconstpool:$Hi, tconstpool:$Lo12, ALIGN),
+                   (ADRPxi tconstpool:$Hi), (i64 tconstpool:$Lo12)>;
 
 // We also want to use uimm12 instructions for local variables at the moment.
 def tframeindex_XFORM : SDNodeXForm<frameindex, [{

lib/Target/AArch64/AArch64MachineFunctionInfo.h

@@ -14,7 +14,6 @@
 #ifndef AARCH64MACHINEFUNCTIONINFO_H
 #define AARCH64MACHINEFUNCTIONINFO_H
 
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/MachineFunction.h"
 
 namespace llvm {
@@ -49,10 +48,6 @@ class AArch64MachineFunctionInfo : public MachineFunctionInfo {
   /// Number of local-dynamic TLS accesses.
   unsigned NumLocalDynamics;
 
-  /// Keep track of the next label to be created within this function to
-  /// represent a cloned constant pool entry. Used by constant islands pass.
-  unsigned PICLabelUId;
-
   /// @see AArch64 Procedure Call Standard, B.3
   ///
   /// The Frame index of the area where LowerFormalArguments puts the
@@ -96,7 +91,6 @@ public:
       ArgumentStackToRestore(0),
       InitialStackAdjust(0),
       NumLocalDynamics(0),
-      PICLabelUId(0),
       VariadicGPRIdx(0),
       VariadicGPRSize(0),
       VariadicFPRIdx(0),
@@ -109,7 +103,6 @@ public:
       ArgumentStackToRestore(0),
       InitialStackAdjust(0),
       NumLocalDynamics(0),
-      PICLabelUId(0),
       VariadicGPRIdx(0),
       VariadicGPRSize(0),
       VariadicFPRIdx(0),
@@ -131,10 +124,6 @@ public:
   unsigned getNumLocalDynamicTLSAccesses() const { return NumLocalDynamics; }
   void incNumLocalDynamicTLSAccesses() { ++NumLocalDynamics; }
 
-  void initPICLabelUId(unsigned UId) { PICLabelUId = UId; }
-  unsigned getNumPICLabels() const { return PICLabelUId; }
-  unsigned createPICLabelUId() { return PICLabelUId++; }
-
   int getVariadicGPRIdx() const { return VariadicGPRIdx; }
   void setVariadicGPRIdx(int Idx) { VariadicGPRIdx = Idx; }
 

lib/Target/AArch64/AArch64TargetMachine.cpp

@@ -66,7 +66,6 @@ TargetPassConfig *AArch64TargetMachine::createPassConfig(PassManagerBase &PM) {
 
 bool AArch64PassConfig::addPreEmitPass() {
   addPass(&UnpackMachineBundlesID);
-  addPass(createAArch64ConstantIslandPass());
   return true;
 }
 

lib/Target/AArch64/CMakeLists.txt

@@ -14,7 +14,6 @@ add_public_tablegen_target(AArch64CommonTableGen)
 
 add_llvm_target(AArch64CodeGen
   AArch64AsmPrinter.cpp
-  AArch64ConstantIslandPass.cpp
   AArch64FrameLowering.cpp
   AArch64ISelDAGToDAG.cpp
   AArch64ISelLowering.cpp

test/CodeGen/AArch64/adrp-relocation.ll

@@ -1,16 +1,16 @@
 ; RUN: llc -mtriple=aarch64-none-linux-gnu -verify-machineinstrs -filetype=obj < %s | elf-dump | FileCheck %s
 
-define fp128 @testfn() nounwind {
+define i64 @testfn() nounwind {
 entry:
-  ret fp128 0xL00000000000000004004500000000000
+  ret i64 0
 }
 
-define fp128 @foo() nounwind {
+define i64 @foo() nounwind {
 entry:
-  %bar = alloca fp128 ()*, align 8
+  %bar = alloca i64 ()*, align 8
-  store fp128 ()* @testfn, fp128 ()** %bar, align 8
+  store i64 ()* @testfn, i64 ()** %bar, align 8
-  %call = call fp128 @testfn()
+  %call = call i64 @testfn()
-  ret fp128 %call
+  ret i64 %call
 }
 
 ; The above should produce an ADRP/ADD pair to calculate the address of
@@ -22,14 +22,14 @@ entry:
 ; CHECK: .rela.text
 
 ; CHECK: # Relocation 0
-; CHECK-NEXT: (('r_offset', 0x0000000000000028)
+; CHECK-NEXT: (('r_offset', 0x0000000000000010)
-; CHECK-NEXT:  ('r_sym', 0x00000009)
+; CHECK-NEXT:  ('r_sym', 0x00000007)
 ; CHECK-NEXT:  ('r_type', 0x00000113)
 ; CHECK-NEXT:  ('r_addend', 0x0000000000000000)
 ; CHECK-NEXT: ),
 ; CHECK-NEXT:  Relocation 1
-; CHECK-NEXT: (('r_offset', 0x000000000000002c)
+; CHECK-NEXT: (('r_offset', 0x0000000000000014)
-; CHECK-NEXT:  ('r_sym', 0x00000009)
+; CHECK-NEXT:  ('r_sym', 0x00000007)
 ; CHECK-NEXT:  ('r_type', 0x00000115)
 ; CHECK-NEXT:  ('r_addend', 0x0000000000000000)
 ; CHECK-NEXT: ),

test/CodeGen/AArch64/extern-weak.ll

@@ -6,8 +6,9 @@ define i32()* @foo() {
 ; The usual ADRP/ADD pair can't be used for a weak reference because it must
 ; evaluate to 0 if the symbol is undefined. We use a litpool entry.
   ret i32()* @var
-; CHECK: ldr x0, .LCPI0_0
-
 ; CHECK: .LCPI0_0:
 ; CHECK-NEXT: .xword var
+
+; CHECK: ldr x0, [{{x[0-9]+}}, #:lo12:.LCPI0_0]
+
 }

test/CodeGen/AArch64/fp-cond-sel.ll

@@ -9,15 +9,15 @@ define void @test_csel(i32 %lhs32, i32 %rhs32, i64 %lhs64) {
   %tst1 = icmp ugt i32 %lhs32, %rhs32
   %val1 = select i1 %tst1, float 0.0, float 1.0
   store float %val1, float* @varfloat
+; CHECK: ldr [[FLT0:s[0-9]+]], [{{x[0-9]+}}, #:lo12:.LCPI
 ; CHECK: fmov [[FLT1:s[0-9]+]], #1.0
-; CHECK: ldr [[FLT0:s[0-9]+]], .LCPI
 ; CHECK: fcsel {{s[0-9]+}}, [[FLT0]], [[FLT1]], hi
 
   %rhs64 = sext i32 %rhs32 to i64
   %tst2 = icmp sle i64 %lhs64, %rhs64
   %val2 = select i1 %tst2, double 1.0, double 0.0
   store double %val2, double* @vardouble
-; CHECK: ldr [[FLT0:d[0-9]+]], .LCPI
+; CHECK: ldr [[FLT0:d[0-9]+]], [{{x[0-9]+}}, #:lo12:.LCPI
 ; CHECK: fmov [[FLT1:d[0-9]+]], #1.0
 ; CHECK: fcsel {{d[0-9]+}}, [[FLT1]], [[FLT0]], le
 

test/CodeGen/AArch64/fp128-folding.ll

@@ -12,6 +12,6 @@ define fp128 @test_folding() {
   %fpval = sitofp i32 %val to fp128
   ; If the value is loaded from a constant pool into an fp128, it's been folded
   ; successfully.
-; CHECK: ldr {{q[0-9]+}}, .LCPI
+; CHECK: ldr {{q[0-9]+}}, [{{x[0-9]+}}, #:lo12:.LCPI
   ret fp128 %fpval
 }

test/CodeGen/AArch64/fp128.ll

@@ -261,6 +261,10 @@ define void @test_extend() {
 }
 
 define fp128 @test_neg(fp128 %in) {
+; CHECK: [[MINUS0:.LCPI[0-9]+_0]]:
+; Make sure the weird hex constant below *is* -0.0
+; CHECK-NEXT: fp128 -0
+
 ; CHECK: test_neg:
 
   ; Could in principle be optimized to fneg which we can't select, this makes
@@ -268,13 +272,9 @@ define fp128 @test_neg(fp128 %in) {
   %ret = fsub fp128 0xL00000000000000008000000000000000, %in
 ; CHECK: str q0, [sp, #-16]
 ; CHECK-NEXT: ldr q1, [sp], #16
-; CHECK: ldr q0, [[MINUS0:.LCPI[0-9]+_0]]
+; CHECK: ldr q0, [{{x[0-9]+}}, #:lo12:[[MINUS0]]]
 ; CHECK: bl __subtf3
 
   ret fp128 %ret
 ; CHECK: ret
-
-; CHECK: [[MINUS0]]:
-; Make sure the weird hex constant below *is* -0.0
-; CHECK-NEXT: fp128 -0
 }

test/CodeGen/AArch64/fpimm.ll

@@ -13,7 +13,7 @@ define void @check_float() {
 
   %newval2 = fadd float %val, 128.0
   store volatile float %newval2, float* @varf32
-; CHECK: ldr {{s[0-9]+}}, .LCPI0_0
+; CHECK: ldr {{s[0-9]+}}, [{{x[0-9]+}}, #:lo12:.LCPI0_0
 
   ret void
 }
@@ -28,7 +28,7 @@ define void @check_double() {
 
   %newval2 = fadd double %val, 128.0
   store volatile double %newval2, double* @varf64
-; CHECK: ldr {{d[0-9]+}}, .LCPI1_0
+; CHECK: ldr {{d[0-9]+}}, [{{x[0-9]+}}, #:lo12:.LCPI1_0
 
   ret void
 }

test/CodeGen/AArch64/func-argpassing.ll

@@ -83,7 +83,7 @@ define i32 @return_int() {
 define double @return_double() {
 ; CHECK: return_double:
     ret double 3.14
-; CHECK: ldr d0, .LCPI
+; CHECK: ldr d0, [{{x[0-9]+}}, #:lo12:.LCPI
 }
 
 ; This is the kind of IR clang will produce for returning a struct

test/CodeGen/AArch64/func-calls.ll

@@ -90,7 +90,7 @@ define void @check_stack_args() {
   call void @stacked_fpu(float -1.0, double 1.0, float 4.0, float 2.0,
                          float -2.0, float -8.0, float 16.0, float 1.0,
                          float 64.0)
-; CHECK: ldr s[[STACKEDREG:[0-9]+]], .LCPI
+; CHECK: ldr s[[STACKEDREG:[0-9]+]], [{{x[0-9]+}}, #:lo12:.LCPI
 ; CHECK: mov x0, sp
 ; CHECK: str d[[STACKEDREG]], [x0]
 ; CHECK bl stacked_fpu

test/CodeGen/AArch64/literal_pools.ll

@@ -10,19 +10,23 @@ define void @foo() {
 
     %val32_lit32 = and i32 %val32, 123456785
     store volatile i32 %val32_lit32, i32* @var32
-; CHECK: ldr {{w[0-9]+}}, .LCPI0
+; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI0_[0-9]+]]
+; CHECK: ldr {{w[0-9]+}}, [x[[LITBASE]], #:lo12:[[CURLIT]]]
 
     %val64_lit32 = and i64 %val64, 305402420
     store volatile i64 %val64_lit32, i64* @var64
-; CHECK: ldr {{w[0-9]+}}, .LCPI0
+; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI0_[0-9]+]]
+; CHECK: ldr {{w[0-9]+}}, [x[[LITBASE]], #:lo12:[[CURLIT]]]
 
     %val64_lit32signed = and i64 %val64, -12345678
     store volatile i64 %val64_lit32signed, i64* @var64
-; CHECK: ldrsw {{x[0-9]+}}, .LCPI0
+; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI0_[0-9]+]]
+; CHECK: ldrsw {{x[0-9]+}}, [x[[LITBASE]], #:lo12:[[CURLIT]]]
 
     %val64_lit64 = and i64 %val64, 1234567898765432
     store volatile i64 %val64_lit64, i64* @var64
-; CHECK: ldr {{x[0-9]+}}, .LCPI0
+; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI0_[0-9]+]]
+; CHECK: ldr {{x[0-9]+}}, [x[[LITBASE]], #:lo12:[[CURLIT]]]
 
     ret void
 }
@@ -35,13 +39,15 @@ define void @floating_lits() {
 
   %floatval = load float* @varfloat
   %newfloat = fadd float %floatval, 128.0
-; CHECK: ldr {{s[0-9]+}}, .LCPI1
+; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI1_[0-9]+]]
+; CHECK: ldr {{s[0-9]+}}, [x[[LITBASE]], #:lo12:[[CURLIT]]]
 ; CHECK: fadd
   store float %newfloat, float* @varfloat
 
   %doubleval = load double* @vardouble
   %newdouble = fadd double %doubleval, 129.0
-; CHECK: ldr {{d[0-9]+}}, .LCPI1
+; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI1_[0-9]+]]
+; CHECK: ldr {{d[0-9]+}}, [x[[LITBASE]], #:lo12:[[CURLIT]]]
 ; CHECK: fadd
   store double %newdouble, double* @vardouble