[AArch64][GlobalISel] Add support for extending indexed loads. (#70373)

2026-01-26 03:56:16 +08:00 · 2023-10-26 13:38:09 -07:00
parent e3ecdf7d3b
commit c9e8b73694
5 changed files with 269 additions and 273 deletions
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -230,6 +230,7 @@ private:
  bool selectMOPS(MachineInstr &I, MachineRegisterInfo &MRI);
  bool selectUSMovFromExtend(MachineInstr &I, MachineRegisterInfo &MRI);

+  bool selectIndexedExtLoad(MachineInstr &I, MachineRegisterInfo &MRI);
  bool selectIndexedLoad(MachineInstr &I, MachineRegisterInfo &MRI);
  bool selectIndexedStore(GIndexedStore &I, MachineRegisterInfo &MRI);

@@ -3047,6 +3048,9 @@ bool AArch64InstructionSelector::select(MachineInstr &I) {
    return constrainSelectedInstRegOperands(*LoadStore, TII, TRI, RBI);
  }

+  case TargetOpcode::G_INDEXED_ZEXTLOAD:
+  case TargetOpcode::G_INDEXED_SEXTLOAD:
+    return selectIndexedExtLoad(I, MRI);
  case TargetOpcode::G_INDEXED_LOAD:
    return selectIndexedLoad(I, MRI);
  case TargetOpcode::G_INDEXED_STORE:
@@ -5648,6 +5652,93 @@ MachineInstr *AArch64InstructionSelector::tryAdvSIMDModImmFP(
  return &*Mov;
 }

+bool AArch64InstructionSelector::selectIndexedExtLoad(
+    MachineInstr &MI, MachineRegisterInfo &MRI) {
+  auto &ExtLd = cast<GIndexedExtLoad>(MI);
+  Register Dst = ExtLd.getDstReg();
+  Register WriteBack = ExtLd.getWritebackReg();
+  Register Base = ExtLd.getBaseReg();
+  Register Offset = ExtLd.getOffsetReg();
+  LLT Ty = MRI.getType(Dst);
+  assert(Ty.getSizeInBits() <= 64); // Only for scalar GPRs.
+  unsigned MemSizeBits = ExtLd.getMMO().getMemoryType().getSizeInBits();
+  bool IsPre = ExtLd.isPre();
+  bool IsSExt = isa<GIndexedSExtLoad>(ExtLd);
+  bool InsertIntoXReg = false;
+  bool IsDst64 = Ty.getSizeInBits() == 64;
+
+  unsigned Opc = 0;
+  LLT NewLdDstTy;
+  LLT s32 = LLT::scalar(32);
+  LLT s64 = LLT::scalar(64);
+
+  if (MemSizeBits == 8) {
+    if (IsSExt) {
+      if (IsDst64)
+        Opc = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
+      else
+        Opc = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
+      NewLdDstTy = IsDst64 ? s64 : s32;
+    } else {
+      Opc = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
+      InsertIntoXReg = IsDst64;
+      NewLdDstTy = s32;
+    }
+  } else if (MemSizeBits == 16) {
+    if (IsSExt) {
+      if (IsDst64)
+        Opc = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
+      else
+        Opc = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
+      NewLdDstTy = IsDst64 ? s64 : s32;
+    } else {
+      Opc = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
+      InsertIntoXReg = IsDst64;
+      NewLdDstTy = s32;
+    }
+  } else if (MemSizeBits == 32) {
+    if (IsSExt) {
+      Opc = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
+      NewLdDstTy = s64;
+    } else {
+      Opc = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
+      InsertIntoXReg = IsDst64;
+      NewLdDstTy = s32;
+    }
+  } else {
+    llvm_unreachable("Unexpected size for indexed load");
+  }
+
+  if (RBI.getRegBank(Dst, MRI, TRI)->getID() == AArch64::FPRRegBankID)
+    return false; // We should be on gpr.
+
+  auto Cst = getIConstantVRegVal(Offset, MRI);
+  if (!Cst)
+    return false; // Shouldn't happen, but just in case.
+
+  auto LdMI = MIB.buildInstr(Opc, {WriteBack, NewLdDstTy}, {Base})
+                  .addImm(Cst->getSExtValue());
+  LdMI.cloneMemRefs(ExtLd);
+  constrainSelectedInstRegOperands(*LdMI, TII, TRI, RBI);
+  // Make sure to select the load with the MemTy as the dest type, and then
+  // insert into X reg if needed.
+  if (InsertIntoXReg) {
+    // Generate a SUBREG_TO_REG.
+    auto SubToReg = MIB.buildInstr(TargetOpcode::SUBREG_TO_REG, {Dst}, {})
+                        .addImm(0)
+                        .addUse(LdMI.getReg(1))
+                        .addImm(AArch64::sub_32);
+    RBI.constrainGenericRegister(SubToReg.getReg(0), AArch64::GPR64RegClass,
+                                 MRI);
+  } else {
+    auto Copy = MIB.buildCopy(Dst, LdMI.getReg(1));
+    selectCopy(*Copy, TII, MRI, TRI, RBI);
+  }
+  MI.eraseFromParent();
+
+  return true;
+}
+
 bool AArch64InstructionSelector::selectIndexedLoad(MachineInstr &MI,
                                                   MachineRegisterInfo &MRI) {
  // TODO: extending loads.
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -459,7 +459,24 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
      .legalIf(IndexedLoadBasicPred)
      .unsupported();
  getActionDefinitionsBuilder({G_INDEXED_SEXTLOAD, G_INDEXED_ZEXTLOAD})
-      .unsupported(); // TODO: implement
+      .unsupportedIf(
+          atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Unordered))
+      .legalIf(all(typeInSet(0, {s16, s32, s64}),
+                   LegalityPredicate([=](const LegalityQuery &Q) {
+                     LLT LdTy = Q.Types[0];
+                     LLT PtrTy = Q.Types[1];
+                     LLT MemTy = Q.MMODescrs[0].MemoryTy;
+                     if (PtrTy != p0)
+                       return false;
+                     if (LdTy == s16)
+                       return MemTy == s8;
+                     if (LdTy == s32)
+                       return MemTy == s8 || MemTy == s16;
+                     if (LdTy == s64)
+                       return MemTy == s8 || MemTy == s16 || MemTy == s32;
+                     return false;
+                   })))
+      .unsupported();

  // Constants
  getActionDefinitionsBuilder(G_CONSTANT)
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -887,9 +887,12 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
      break;
    }
    break;
-  case TargetOpcode::G_INDEXED_LOAD:
  case TargetOpcode::G_INDEXED_SEXTLOAD:
-  case TargetOpcode::G_INDEXED_ZEXTLOAD: {
+  case TargetOpcode::G_INDEXED_ZEXTLOAD:
+    // These should always be GPR.
+    OpRegBankIdx[0] = PMI_FirstGPR;
+    break;
+  case TargetOpcode::G_INDEXED_LOAD: {
    if (isLoadFromFPType(MI))
      OpRegBankIdx[0] = PMI_FirstFPR;
    break;
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-indexed-load-stores.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-indexed-load-stores.mir
@@ -109,3 +109,83 @@ body: |
    $x0 = COPY %writeback
    RET_ReallyLR implicit $x0
 ...
+---
+name:            post_zextload_s8_to_s64
+body: |
+  bb.0:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: post_zextload_s8_to_s64
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+    ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8
+    ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset(s64), 0 :: (load (s8), align 8)
+    ; CHECK-NEXT: $x0 = COPY %writeback(p0)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %ptr:_(p0) = COPY $x0
+    %offset:_(s64) = G_CONSTANT i64 8
+    %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset, 0 :: (load (s8), align 8)
+    $x0 = COPY %writeback
+    RET_ReallyLR implicit $x0
+...
+---
+name:            post_sextload_s8_to_s64
+body: |
+  bb.0:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: post_sextload_s8_to_s64
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+    ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8
+    ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset(s64), 0 :: (load (s8), align 8)
+    ; CHECK-NEXT: $x0 = COPY %writeback(p0)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %ptr:_(p0) = COPY $x0
+    %offset:_(s64) = G_CONSTANT i64 8
+    %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset, 0 :: (load (s8), align 8)
+    $x0 = COPY %writeback
+    RET_ReallyLR implicit $x0
+...
+---
+name:            post_sextload_s32_to_s64
+body: |
+  bb.0:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: post_sextload_s32_to_s64
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+    ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8
+    ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset(s64), 0 :: (load (s32), align 8)
+    ; CHECK-NEXT: $x0 = COPY %writeback(p0)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %ptr:_(p0) = COPY $x0
+    %offset:_(s64) = G_CONSTANT i64 8
+    %dst:_(s64), %writeback:_(p0) = G_INDEXED_SEXTLOAD %ptr, %offset, 0 :: (load (s32), align 8)
+    $x0 = COPY %writeback
+    RET_ReallyLR implicit $x0
+...
+---
+name:            post_zextload_s32_to_s64
+body: |
+  bb.0:
+    liveins: $x0
+
+    ; CHECK-LABEL: name: post_zextload_s32_to_s64
+    ; CHECK: liveins: $x0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %ptr:_(p0) = COPY $x0
+    ; CHECK-NEXT: %offset:_(s64) = G_CONSTANT i64 8
+    ; CHECK-NEXT: %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset(s64), 0 :: (load (s32), align 8)
+    ; CHECK-NEXT: $x0 = COPY %writeback(p0)
+    ; CHECK-NEXT: RET_ReallyLR implicit $x0
+    %ptr:_(p0) = COPY $x0
+    %offset:_(s64) = G_CONSTANT i64 8
+    %dst:_(s64), %writeback:_(p0) = G_INDEXED_ZEXTLOAD %ptr, %offset, 0 :: (load (s32), align 8)
+    $x0 = COPY %writeback
+    RET_ReallyLR implicit $x0
+...
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
@@ -503,24 +503,11 @@ define ptr @preidx8zext64(ptr %src, ptr %out) {
 }

 define ptr @preidx32sext64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx32sext64:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsw x8, [x0, #4]!
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx32sext64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsw x8, [x0, #4]
-; GISEL-NEXT:    add x0, x0, #4
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx32sext64:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsw x8, [x0, #4]!
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx32sext64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsw x8, [x0, #4]!
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i32, ptr %src, i64 1
  %tmp = load i32, ptr %ptr, align 4
  %ext = sext i32 %tmp to i64
@@ -529,24 +516,11 @@ define ptr @preidx32sext64(ptr %src, ptr %out) {
 }

 define ptr @preidx16sext32(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx16sext32:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsh w8, [x0, #2]!
-; CHECK64-NEXT:    str w8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx16sext32:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsh w8, [x0, #2]
-; GISEL-NEXT:    add x0, x0, #2
-; GISEL-NEXT:    str w8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx16sext32:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsh w8, [x0, #2]!
-; CHECK32-NEXT:    str w8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx16sext32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsh w8, [x0, #2]!
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i16, ptr %src, i64 1
  %tmp = load i16, ptr %ptr, align 4
  %ext = sext i16 %tmp to i32
@@ -555,24 +529,11 @@ define ptr @preidx16sext32(ptr %src, ptr %out) {
 }

 define ptr @preidx16sext64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx16sext64:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsh x8, [x0, #2]!
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx16sext64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsh x8, [x0, #2]
-; GISEL-NEXT:    add x0, x0, #2
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx16sext64:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsh x8, [x0, #2]!
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx16sext64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsh x8, [x0, #2]!
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i16, ptr %src, i64 1
  %tmp = load i16, ptr %ptr, align 4
  %ext = sext i16 %tmp to i64
@@ -581,24 +542,11 @@ define ptr @preidx16sext64(ptr %src, ptr %out) {
 }

 define ptr @preidx8sext32(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx8sext32:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsb w8, [x0, #1]!
-; CHECK64-NEXT:    str w8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx8sext32:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsb w8, [x0, #1]
-; GISEL-NEXT:    add x0, x0, #1
-; GISEL-NEXT:    str w8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx8sext32:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsb w8, [x0, #1]!
-; CHECK32-NEXT:    str w8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx8sext32:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsb w8, [x0, #1]!
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i8, ptr %src, i64 1
  %tmp = load i8, ptr %ptr, align 4
  %ext = sext i8 %tmp to i32
@@ -607,24 +555,11 @@ define ptr @preidx8sext32(ptr %src, ptr %out) {
 }

 define ptr @preidx8sext64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx8sext64:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsb x8, [x0, #1]!
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx8sext64:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsb x8, [x0, #1]
-; GISEL-NEXT:    add x0, x0, #1
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx8sext64:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsb x8, [x0, #1]!
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx8sext64:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsb x8, [x0, #1]!
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i8, ptr %src, i64 1
  %tmp = load i8, ptr %ptr, align 4
  %ext = sext i8 %tmp to i64
@@ -662,24 +597,11 @@ define ptr @postidx_clobber(ptr %addr) nounwind noinline ssp {
 }

 define ptr @preidx32_sb(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx32_sb:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsb w8, [x0, #1]!
-; CHECK64-NEXT:    str w8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx32_sb:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsb w8, [x0, #1]
-; GISEL-NEXT:    add x0, x0, #1
-; GISEL-NEXT:    str w8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx32_sb:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsb w8, [x0, #1]!
-; CHECK32-NEXT:    str w8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx32_sb:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsb w8, [x0, #1]!
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i8, ptr %src, i64 1
  %tmp = load i8, ptr %ptr, align 1
  %sext = sext i8 %tmp to i32
@@ -688,24 +610,11 @@ define ptr @preidx32_sb(ptr %src, ptr %out) {
 }

 define ptr @preidx32_sh(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx32_sh:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsh w8, [x0, #2]!
-; CHECK64-NEXT:    str w8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx32_sh:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsh w8, [x0, #2]
-; GISEL-NEXT:    add x0, x0, #2
-; GISEL-NEXT:    str w8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx32_sh:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsh w8, [x0, #2]!
-; CHECK32-NEXT:    str w8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx32_sh:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsh w8, [x0, #2]!
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i16, ptr %src, i64 1
  %tmp = load i16, ptr %ptr, align 2
  %sext = sext i16 %tmp to i32
@@ -714,24 +623,11 @@ define ptr @preidx32_sh(ptr %src, ptr %out) {
 }

 define ptr @preidx64_sb(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx64_sb:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsb x8, [x0, #1]!
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx64_sb:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsb x8, [x0, #1]
-; GISEL-NEXT:    add x0, x0, #1
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx64_sb:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsb x8, [x0, #1]!
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx64_sb:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsb x8, [x0, #1]!
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i8, ptr %src, i64 1
  %tmp = load i8, ptr %ptr, align 1
  %sext = sext i8 %tmp to i64
@@ -740,24 +636,11 @@ define ptr @preidx64_sb(ptr %src, ptr %out) {
 }

 define ptr @preidx64_sh(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx64_sh:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsh x8, [x0, #2]!
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx64_sh:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsh x8, [x0, #2]
-; GISEL-NEXT:    add x0, x0, #2
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx64_sh:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsh x8, [x0, #2]!
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx64_sh:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsh x8, [x0, #2]!
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i16, ptr %src, i64 1
  %tmp = load i16, ptr %ptr, align 2
  %sext = sext i16 %tmp to i64
@@ -766,24 +649,11 @@ define ptr @preidx64_sh(ptr %src, ptr %out) {
 }

 define ptr @preidx64_sw(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx64_sw:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsw x8, [x0, #4]!
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: preidx64_sw:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsw x8, [x0, #4]
-; GISEL-NEXT:    add x0, x0, #4
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: preidx64_sw:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsw x8, [x0, #4]!
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: preidx64_sw:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsw x8, [x0, #4]!
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %ptr = getelementptr inbounds i32, ptr %src, i64 1
  %tmp = load i32, ptr %ptr, align 2
  %sext = sext i32 %tmp to i64
@@ -792,24 +662,11 @@ define ptr @preidx64_sw(ptr %src, ptr %out) {
 }

 define ptr @postidx32_sb(ptr %src, ptr %out) {
-; CHECK64-LABEL: postidx32_sb:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsb w8, [x0], #1
-; CHECK64-NEXT:    str w8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: postidx32_sb:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsb w8, [x0]
-; GISEL-NEXT:    add x0, x0, #1
-; GISEL-NEXT:    str w8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: postidx32_sb:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsb w8, [x0], #1
-; CHECK32-NEXT:    str w8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: postidx32_sb:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsb w8, [x0], #1
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
  %tmp = load i8, ptr %src, align 1
  %ptr = getelementptr inbounds i8, ptr %src, i64 1
  %sext = sext i8 %tmp to i32
@@ -818,24 +675,11 @@ define ptr @postidx32_sb(ptr %src, ptr %out) {
 }

 define ptr @postidx32_sh(ptr %src, ptr %out) {
-; CHECK64-LABEL: postidx32_sh:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsh w8, [x0], #2
-; CHECK64-NEXT:    str w8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: postidx32_sh:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsh w8, [x0]
-; GISEL-NEXT:    add x0, x0, #2
-; GISEL-NEXT:    str w8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: postidx32_sh:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsh w8, [x0], #2
-; CHECK32-NEXT:    str w8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: postidx32_sh:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsh w8, [x0], #2
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
  %tmp = load i16, ptr %src, align 2
  %ptr = getelementptr inbounds i16, ptr %src, i64 1
  %sext = sext i16 %tmp to i32
@@ -844,24 +688,11 @@ define ptr @postidx32_sh(ptr %src, ptr %out) {
 }

 define ptr @postidx64_sb(ptr %src, ptr %out) {
-; CHECK64-LABEL: postidx64_sb:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsb x8, [x0], #1
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: postidx64_sb:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsb x8, [x0]
-; GISEL-NEXT:    add x0, x0, #1
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: postidx64_sb:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsb x8, [x0], #1
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: postidx64_sb:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsb x8, [x0], #1
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %tmp = load i8, ptr %src, align 1
  %ptr = getelementptr inbounds i8, ptr %src, i64 1
  %sext = sext i8 %tmp to i64
@@ -870,24 +701,11 @@ define ptr @postidx64_sb(ptr %src, ptr %out) {
 }

 define ptr @postidx64_sh(ptr %src, ptr %out) {
-; CHECK64-LABEL: postidx64_sh:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsh x8, [x0], #2
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: postidx64_sh:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsh x8, [x0]
-; GISEL-NEXT:    add x0, x0, #2
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: postidx64_sh:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsh x8, [x0], #2
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: postidx64_sh:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsh x8, [x0], #2
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %tmp = load i16, ptr %src, align 2
  %ptr = getelementptr inbounds i16, ptr %src, i64 1
  %sext = sext i16 %tmp to i64
@@ -896,24 +714,11 @@ define ptr @postidx64_sh(ptr %src, ptr %out) {
 }

 define ptr @postidx64_sw(ptr %src, ptr %out) {
-; CHECK64-LABEL: postidx64_sw:
-; CHECK64:       ; %bb.0:
-; CHECK64-NEXT:    ldrsw x8, [x0], #4
-; CHECK64-NEXT:    str x8, [x1]
-; CHECK64-NEXT:    ret
-;
-; GISEL-LABEL: postidx64_sw:
-; GISEL:       ; %bb.0:
-; GISEL-NEXT:    ldrsw x8, [x0]
-; GISEL-NEXT:    add x0, x0, #4
-; GISEL-NEXT:    str x8, [x1]
-; GISEL-NEXT:    ret
-;
-; CHECK32-LABEL: postidx64_sw:
-; CHECK32:       ; %bb.0:
-; CHECK32-NEXT:    ldrsw x8, [x0], #4
-; CHECK32-NEXT:    str x8, [x1]
-; CHECK32-NEXT:    ret
+; CHECK-LABEL: postidx64_sw:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrsw x8, [x0], #4
+; CHECK-NEXT:    str x8, [x1]
+; CHECK-NEXT:    ret
  %tmp = load i32, ptr %src, align 4
  %ptr = getelementptr inbounds i32, ptr %src, i64 1
  %sext = sext i32 %tmp to i64