diff --git a/gcc/config/rs6000/rs6000-string.cc b/gcc/config/rs6000/rs6000-string.cc
index f707bb2727e..018b87f2501 100644
--- a/gcc/config/rs6000/rs6000-string.cc
+++ b/gcc/config/rs6000/rs6000-string.cc
@@ -404,21 +404,6 @@ do_ifelse (machine_mode cmpmode, rtx_code comparison,
   LABEL_NUSES (true_label) += 1;
 }

-/* Emit an isel of the proper mode for DEST.
-
-   DEST is the isel destination register.
-   SRC1 is the isel source if CR is true.
-   SRC2 is the isel source if CR is false.
-   CR is the condition for the isel.  */
-static void
-do_isel (rtx dest, rtx cmp, rtx src_t, rtx src_f, rtx cr)
-{
-  if (GET_MODE (dest) == DImode)
-    emit_insn (gen_isel_cc_di (dest, cmp, src_t, src_f, cr));
-  else
-    emit_insn (gen_isel_cc_si (dest, cmp, src_t, src_f, cr));
-}
-
 /* Emit a subtract of the proper mode for DEST.

    DEST is the destination register for the subtract.
@@ -499,65 +484,61 @@ do_rotl3 (rtx dest, rtx src1, rtx src2)
     emit_insn (gen_rotlsi3 (dest, src1, src2));
 }

-/* Generate rtl for a load, shift, and compare of less than a full word.
-
-   LOAD_MODE is the machine mode for the loads.
-   DIFF is the reg for the difference.
-   CMP_REM is the reg containing the remaining bytes to compare.
-   DCOND is the CCUNS reg for the compare if we are doing P9 code with setb.
-   SRC1_ADDR is the first source address.
-   SRC2_ADDR is the second source address.
-   ORIG_SRC1 is the original first source block's address rtx.
-   ORIG_SRC2 is the original second source block's address rtx.  */
+/* Do the compare for two registers.  */
 static void
-do_load_mask_compare (const machine_mode load_mode, rtx diff, rtx cmp_rem, rtx dcond,
-		      rtx src1_addr, rtx src2_addr, rtx orig_src1, rtx orig_src2)
+do_reg_compare (bool use_vec, rtx vec_result, rtx diff, rtx *dcond, rtx d1,
+		rtx d2)
 {
-  HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
-  rtx shift_amount = gen_reg_rtx (word_mode);
-  rtx d1 = gen_reg_rtx (word_mode);
-  rtx d2 = gen_reg_rtx (word_mode);
-
-  do_load_for_compare_from_addr (load_mode, d1, src1_addr, orig_src1);
-  do_load_for_compare_from_addr (load_mode, d2, src2_addr, orig_src2);
-  do_sub3 (shift_amount, GEN_INT (load_mode_size), cmp_rem);
-
-  if (word_mode == DImode)
-    {
-      emit_insn (gen_ashldi3 (shift_amount, shift_amount,
-			      GEN_INT (LOG2_BITS_PER_UNIT)));
-      emit_insn (gen_lshrdi3 (d1, d1,
-			      gen_lowpart (SImode, shift_amount)));
-      emit_insn (gen_lshrdi3 (d2, d2,
-			      gen_lowpart (SImode, shift_amount)));
-    }
-  else
-    {
-      emit_insn (gen_ashlsi3 (shift_amount, shift_amount,
-			      GEN_INT (LOG2_BITS_PER_UNIT)));
-      emit_insn (gen_lshrsi3 (d1, d1, shift_amount));
-      emit_insn (gen_lshrsi3 (d2, d2, shift_amount));
-    }
+  gcc_assert (!use_vec || vec_result != NULL_RTX);
+  gcc_assert (REG_P (d1) && REG_P (d2));
+  gcc_assert (GET_MODE (d1) == GET_MODE (d2));

-  if (TARGET_P9_MISC)
+  if (use_vec)
+    emit_insn (gen_altivec_vcmpequb_p (vec_result, d1, d2));
+  else if (TARGET_P9_MISC)
     {
       /* Generate a compare, and convert with a setb later.  */
       rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1, d2);
-      emit_insn (gen_rtx_SET (dcond, cmp));
+      emit_insn (gen_rtx_SET (*dcond, cmp));
     }
   else
     {
+      *dcond = gen_reg_rtx (CCmode);
       if (word_mode == DImode)
-	emit_insn (gen_subfdi3_carry (diff, d2, d1));
+	emit_insn (gen_subfdi3_carry_dot2 (diff, d2, d1, *dcond));
       else
-	emit_insn (gen_subfsi3_carry (diff, d2, d1));
+	emit_insn (gen_subfsi3_carry_dot2 (diff, d2, d1, *dcond));
     }
 }

+/* Load the memory to register and do the compare.  */
+static void
+do_load_and_compare (machine_mode load_mode, rtx addr1, rtx addr2, rtx *dcond,
+		     rtx diff, rtx orig_src1, rtx orig_src2)
+{
+  rtx d1 = gen_reg_rtx (word_mode);
+  rtx d2 = gen_reg_rtx (word_mode);
+
+  if (MEM_P (addr1))
+    do_load_for_compare (d1, addr1, load_mode);
+  else
+    do_load_for_compare_from_addr (load_mode, d1, addr1, orig_src1);
+
+  if (MEM_P (addr2))
+    do_load_for_compare (d2, addr2, load_mode);
+  else
+    do_load_for_compare_from_addr (load_mode, d2, addr2, orig_src2);
+
+  do_reg_compare (false, NULL_RTX, diff, dcond, d1, d2);
+}
+
 /* Generate rtl for an overlapping load and compare of less than a
    full load_mode.  This assumes that the previous word is part of the
    block being compared so it's ok to back up part of a word so we can
    compare the last unaligned full word that ends at the end of the block.
+   If the remain bytes can be loaded and compared by a small unit with
+   only one instruction, just do the load and compare by the small unit
+   other than the full word overlapping load and compare.

    LOAD_MODE is the machine mode for the loads.
    ISCONST tells whether the remaining length is a constant or in a register.
@@ -569,55 +550,41 @@ do_load_mask_compare (const machine_mode load_mode, rtx diff, rtx cmp_rem, rtx d
    SRC2_ADDR is the second source address.
    ORIG_SRC1 is the original first source block's address rtx.
    ORIG_SRC2 is the original second source block's address rtx.  */
+
 static void
-do_overlap_load_compare (machine_mode load_mode, bool isConst,
-			HOST_WIDE_INT bytes_rem, rtx diff,
-			rtx cmp_rem, rtx dcond, rtx src1_addr, rtx src2_addr,
-			rtx orig_src1, rtx orig_src2)
+do_overlap_load_compare (machine_mode load_mode, HOST_WIDE_INT bytes_rem,
+			 rtx diff, rtx *dcond, rtx orig_src1, rtx orig_src2,
+			 HOST_WIDE_INT length)
 {
   HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
-  HOST_WIDE_INT addr_adj = load_mode_size - bytes_rem;
-  rtx d1 = gen_reg_rtx (word_mode);
-  rtx d2 = gen_reg_rtx (word_mode);
+  gcc_assert (IN_RANGE (bytes_rem, 0, load_mode_size - 1));

   rtx addr1, addr2;
-  if (!isConst || addr_adj)
-    {
-      rtx adj_reg = gen_reg_rtx (word_mode);
-      if (isConst)
-	emit_move_insn (adj_reg, GEN_INT (-addr_adj));
-      else
-	{
-	  rtx reg_lms = gen_reg_rtx (word_mode);
-	  emit_move_insn (reg_lms, GEN_INT (load_mode_size));
-	  do_sub3 (adj_reg, cmp_rem, reg_lms);
-	}

-      addr1 = gen_rtx_PLUS (word_mode, src1_addr, adj_reg);
-      addr2 = gen_rtx_PLUS (word_mode, src2_addr, adj_reg);
-    }
-  else
+  switch (bytes_rem)
     {
-      addr1 = src1_addr;
-      addr2 = src2_addr;
+      case 0:
+	return;
+      case 1:
+	load_mode = QImode;
+	break;
+      case 2:
+	load_mode = HImode;
+	break;
+      case 4:
+	load_mode = SImode;
+	break;
+      case 8:
+	if (TARGET_POWERPC64)
+	  load_mode = DImode;
+	break;
     }

-  do_load_for_compare_from_addr (load_mode, d1, addr1, orig_src1);
-  do_load_for_compare_from_addr (load_mode, d2, addr2, orig_src2);
-
-  if (TARGET_P9_MISC)
-    {
-      /* Generate a compare, and convert with a setb later.  */
-      rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1, d2);
-      emit_insn (gen_rtx_SET (dcond, cmp));
-    }
-  else
-    {
-      if (word_mode == DImode)
-	emit_insn (gen_subfdi3_carry (diff, d2, d1));
-      else
-	emit_insn (gen_subfsi3_carry (diff, d2, d1));
-    }
+  load_mode_size = GET_MODE_SIZE (load_mode);
+  addr1 = adjust_address (orig_src1, load_mode, length - load_mode_size);
+  addr2 = adjust_address (orig_src2, load_mode, length - load_mode_size);
+  do_load_and_compare (load_mode, addr1, addr2, dcond, diff,
+		       orig_src1, orig_src2);
 }

 /* Generate the sequence of compares for strcmp/strncmp using vec/vsx
@@ -889,790 +856,550 @@ emit_final_compare_vec (rtx str1, rtx str2, rtx result,
   return;
 }

-/* Expand a block compare operation using loop code, and return true
-   if successful.  Return false if we should let the compiler generate
-   normal code, probably a memcmp call.
-
-   OPERANDS[0] is the target (result).
-   OPERANDS[1] is the first source.
-   OPERANDS[2] is the second source.
-   OPERANDS[3] is the length.
-   OPERANDS[4] is the alignment.  */
-bool
-expand_compare_loop (rtx operands[])
+static HOST_WIDE_INT
+get_max_inline_loop_bytes (bool bytes_is_const, int align)
 {
-  rtx target = operands[0];
-  rtx orig_src1 = operands[1];
-  rtx orig_src2 = operands[2];
-  rtx bytes_rtx = operands[3];
-  rtx align_rtx = operands[4];
-
-  /* This case is complicated to handle because the subtract
-     with carry instructions do not generate the 64-bit
-     carry and so we must emit code to calculate it ourselves.
-     We choose not to implement this yet.  */
-  if (TARGET_32BIT && TARGET_POWERPC64)
-    return false;
-
-  /* Allow non-const length.  */
-  int bytes_is_const = CONST_INT_P (bytes_rtx);
-
-  /* This must be a fixed size alignment.  */
-  if (!CONST_INT_P (align_rtx))
-    return false;
-
-  HOST_WIDE_INT align1 = MEM_ALIGN (orig_src1) / BITS_PER_UNIT;
-  HOST_WIDE_INT align2 = MEM_ALIGN (orig_src2) / BITS_PER_UNIT;
-  HOST_WIDE_INT minalign = MIN (align1, align2);
-
-  bool isP7 = (rs6000_tune == PROCESSOR_POWER7);
-
-  gcc_assert (GET_MODE (target) == SImode);
-
-  /* Anything to move?	*/
-  HOST_WIDE_INT bytes = 0;
-  if (bytes_is_const)
-    bytes = INTVAL (bytes_rtx);
-
-  if (bytes_is_const && bytes == 0)
-    return true;
-
-  /* Limit the amount we compare, if known statically.  */
-  HOST_WIDE_INT max_bytes;
   switch (rs6000_tune)
     {
     case PROCESSOR_POWER7:
       if (!bytes_is_const)
-	if (minalign < 8)
-	  max_bytes = 0;
+	if (align < 8)
+	  return 0;
 	else
-	  max_bytes = 128;
+	  return 128;
       else
-	if (minalign < 8)
-	  max_bytes = 32;
+	if (align < 8)
+	  return 32;
 	else
-	  max_bytes = 128;
+	  return 128;
       break;
     case PROCESSOR_POWER8:
       if (!bytes_is_const)
-	max_bytes = 0;
+	return 0;
       else
-	if (minalign < 8)
-	  max_bytes = 128;
+	if (align < 8)
+	  return 128;
 	else
-	  max_bytes = 64;
+	  return 64;
       break;
     case PROCESSOR_POWER9:
     case PROCESSOR_POWER10:
       if (bytes_is_const)
-	max_bytes = 191;
+	return 191;
       else
-	max_bytes = 0;
+	return 0;
       break;
     default:
-      max_bytes = 128;
+      return 128;
     }
+}

-  /* Allow the option to override the default.  */
-  if (rs6000_block_compare_inline_loop_limit >= 0)
-    max_bytes = (unsigned HOST_WIDE_INT) rs6000_block_compare_inline_loop_limit;
-
-  if (max_bytes == 0)
-    return false;
+/* Do the load and compare when remain bytes is less than loop bytes
+   and it's a variable length compare.  expand_bytes indicates the
+   maximum bytes needed to be expanded.  */
+static void
+do_load_compare_rest_of_loop (machine_mode load_mode, rtx src1_addr,
+			      rtx src2_addr, rtx cmp_rem, rtx diff,
+			      rtx diff_label, rtx *dcond, rtx final_label,
+			      rtx orig_src1, rtx orig_src2,
+			      HOST_WIDE_INT loop_bytes,
+			      HOST_WIDE_INT expand_bytes)
+{
+  gcc_assert ((TARGET_POWERPC64 && load_mode == DImode)
+	      || (!TARGET_POWERPC64 && load_mode == SImode));
+  HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
+  gcc_assert (loop_bytes = 2 * load_mode_size);
+  gcc_assert (expand_bytes < loop_bytes);

-  rtx cmp_rem = gen_reg_rtx (word_mode);  /* Remainder for library call.  */
-  rtx loop_cmp = gen_reg_rtx (word_mode); /* Actual amount compared by loop.  */
-  HOST_WIDE_INT niter;
-  rtx iter = gen_reg_rtx (word_mode);
-  rtx iv1 = gen_reg_rtx (word_mode);
-  rtx iv2 = gen_reg_rtx (word_mode);
-  rtx d1_1 = gen_reg_rtx (word_mode);  /* Addr expression src1+iv1 */
-  rtx d1_2 = gen_reg_rtx (word_mode);  /* Addr expression src1+iv2 */
-  rtx d2_1 = gen_reg_rtx (word_mode);  /* Addr expression src2+iv1 */
-  rtx d2_2 = gen_reg_rtx (word_mode);  /* Addr expression src2+iv2 */
+  rtx adj_reg = gen_reg_rtx (word_mode);
+  rtx final_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
+  rtx j;
+  rtx cmp;
+  rtx ccreg = gen_reg_rtx (CCmode);

-  /* Strip unneeded subreg from length if there is one.  */
-  if (SUBREG_P (bytes_rtx) && subreg_lowpart_p (bytes_rtx))
-    bytes_rtx = SUBREG_REG (bytes_rtx);
-  /* Extend bytes_rtx to word_mode if needed.  But, we expect only to
-   maybe have to deal with the case were bytes_rtx is SImode and
-   word_mode is DImode.  */
-  if (!bytes_is_const)
+  if (TARGET_POWERPC64 && expand_bytes >= 8)
     {
-      if (GET_MODE_SIZE (GET_MODE (bytes_rtx)) > GET_MODE_SIZE (word_mode))
-	/* Do not expect length longer than word_mode.  */
-	return false;
-      else if (GET_MODE_SIZE (GET_MODE (bytes_rtx)) < GET_MODE_SIZE (word_mode))
+      /* Compare with 8 bytes.  */
+      rtx cmp_4 = gen_label_rtx ();
+      cmp = gen_rtx_COMPARE (CCmode, cmp_rem, GEN_INT (8));
+      emit_insn (gen_rtx_SET (ccreg, cmp));
+      do_ifelse (CCmode, LT, NULL_RTX, NULL_RTX, ccreg, cmp_4,
+		 profile_probability::even ());
+      do_load_and_compare (DImode, src1_addr, src2_addr, dcond, diff,
+			   orig_src1, orig_src2);
+      do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+		 *dcond, diff_label, profile_probability::unlikely ());
+
+      if (expand_bytes > 8)
 	{
-	  bytes_rtx = force_reg (GET_MODE (bytes_rtx), bytes_rtx);
-	  bytes_rtx = force_reg (word_mode,
-				 gen_rtx_fmt_e (ZERO_EXTEND, word_mode,
-						bytes_rtx));
+	  do_ifelse (CCmode, EQ, NULL_RTX, NULL_RTX, ccreg, final_label,
+		     profile_probability::unlikely ());
+
+	  /* cmp_rem is great than 8 bytes.  Do 8 bytes overlap compare.  */
+	  do_add3 (adj_reg, cmp_rem, GEN_INT (-8));
+	  do_add3 (src1_addr, src1_addr, adj_reg);
+	  do_add3 (src2_addr, src2_addr, adj_reg);
+	  do_load_and_compare (DImode, src1_addr, src2_addr, dcond, diff,
+			       orig_src1, orig_src2);
+	  do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+		     *dcond, diff_label, profile_probability::likely ());
 	}
-      else
-	/* Make sure it's in a register before we get started.  */
-	bytes_rtx = force_reg (GET_MODE (bytes_rtx), bytes_rtx);
-    }
-
-  machine_mode load_mode = word_mode;
-  HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
-
-  /* Number of bytes per iteration of the unrolled loop.  */
-  HOST_WIDE_INT loop_bytes = 2 * load_mode_size;
-  /* max iters and bytes compared in the loop.  */
-  HOST_WIDE_INT max_loop_iter = max_bytes / loop_bytes;
-  HOST_WIDE_INT max_loop_bytes = max_loop_iter * loop_bytes;
-  int l2lb = floor_log2 (loop_bytes);

-  if (bytes_is_const && (max_bytes < load_mode_size
-			 || !IN_RANGE (bytes, load_mode_size, max_bytes)))
-    return false;
+      j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
+      JUMP_LABEL (j) = final_label;
+      LABEL_NUSES (final_label) += 1;
+      emit_barrier ();

-  bool no_remainder_code = false;
-  rtx final_label = gen_label_rtx ();
-  rtx final_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
-  rtx diff_label = gen_label_rtx ();
-  rtx library_call_label = NULL;
-  rtx cleanup_label = gen_label_rtx ();
+      emit_label (cmp_4);
+    }

-  rtx cr;
+  if (expand_bytes >= 4)
+    {
+      /* Compare with 4 bytes.  */
+      rtx cmp_2 = gen_label_rtx ();
+      cmp = gen_rtx_COMPARE (CCmode, cmp_rem, GEN_INT (4));
+      emit_insn (gen_rtx_SET (ccreg, cmp));
+      do_ifelse (CCmode, LT, NULL_RTX, NULL_RTX, ccreg, cmp_2,
+		 profile_probability::even ());
+      do_load_and_compare (SImode, src1_addr, src2_addr, dcond, diff,
+			   orig_src1, orig_src2);
+      do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+		 *dcond, diff_label, profile_probability::unlikely ());
+
+      if (expand_bytes > 4)
+	{
+	  do_ifelse (CCmode, EQ, NULL_RTX, NULL_RTX, ccreg, final_label,
+		     profile_probability::unlikely ());
+
+	  /* cmp_rem is great than 4 bytes.  Do 4 bytes overlap compare.  */
+	  do_add3 (adj_reg, cmp_rem, GEN_INT (-4));
+	  do_add3 (src1_addr, src1_addr, adj_reg);
+	  do_add3 (src2_addr, src2_addr, adj_reg);
+	  do_load_and_compare (SImode, src1_addr, src2_addr, dcond, diff,
+			       orig_src1, orig_src2);
+	  do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+		     *dcond, diff_label, profile_probability::likely ());
+	}

-  rtx src1_addr = copy_addr_to_reg (XEXP (orig_src1, 0));
-  rtx src2_addr = copy_addr_to_reg (XEXP (orig_src2, 0));
+      j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
+      JUMP_LABEL (j) = final_label;
+      LABEL_NUSES (final_label) += 1;
+      emit_barrier ();

-  /* Difference found is stored here before jump to diff_label.  */
-  rtx diff = gen_reg_rtx (word_mode);
-  rtx_insn *j;
+      emit_label (cmp_2);
+  }

-  /* Example of generated code for 35 bytes aligned 1 byte.
-
-	     mtctr 8
-	     li 6,0
-	     li 5,8
-     .L13:
-	     ldbrx 7,3,6
-	     ldbrx 9,10,6
-	     ldbrx 0,3,5
-	     ldbrx 4,10,5
-	     addi 6,6,16
-	     addi 5,5,16
-	     subfc. 9,9,7
-	     bne 0,.L10
-	     subfc. 9,4,0
-	     bdnzt 2,.L13
-	     bne 0,.L10
-	     add 3,3,6
-	     add 10,10,6
-	     addi 9,3,-5
-	     ldbrx 7,0,9
-	     addi 9,10,-5
-	     ldbrx 9,0,9
-	     subfc 9,9,7
-	     .p2align 4,,15
-     .L10:
-	     popcntd 9,9
-	     subfe 10,10,10
-	     or 9,9,10
-
-     Compiled with -fno-reorder-blocks for clarity.  */
-
-  /* Structure of what we're going to do:
-     Two separate lengths: what we will compare before bailing to library
-	call (max_bytes), and the total length to be checked.
-     if length <= 16, branch to linear cleanup code starting with
-	remainder length check (length not known at compile time)
-     set up 2 iv's and load count reg, compute remainder length
-     unrollx2 compare loop
-     if loop exit due to a difference, branch to difference handling code
-     if remainder length < 8, branch to final cleanup compare
-     load and compare 8B
-     final cleanup comparison (depends on alignment and length)
-	load 8B, shift off bytes past length, compare
-	load 8B ending at last byte and compare
-	load/compare 1 byte at a time (short block abutting 4k boundary)
-     difference handling, 64->32 conversion
-     final result
-     branch around memcmp call
-     memcmp library call
-  */
-
-  /* If bytes is not const, compare length and branch directly
-     to the cleanup code that can handle 0-16 bytes if length
-     is >= 16.  Stash away bytes-max_bytes for the library call.  */
-  if (bytes_is_const)
+  if (expand_bytes >= 2)
     {
-      /* These need to be set for some of the places we may jump to.  */
-      if (bytes > max_bytes)
-	{
-	  no_remainder_code = true;
-	  niter = max_loop_iter;
-	  library_call_label = gen_label_rtx ();
-	}
-      else
+      /* Compare with 2 bytes.  */
+      rtx cmp_1 = gen_label_rtx ();
+      cmp = gen_rtx_COMPARE (CCmode, cmp_rem, GEN_INT (2));
+      emit_insn (gen_rtx_SET (ccreg, cmp));
+      do_ifelse (CCmode, LT, NULL_RTX, NULL_RTX, ccreg, cmp_1,
+		 profile_probability::even ());
+      do_load_and_compare (HImode, src1_addr, src2_addr, dcond, diff,
+			   orig_src1, orig_src2);
+      do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+		 *dcond, diff_label, profile_probability::unlikely ());
+
+      if (expand_bytes > 2)
 	{
-	  niter = bytes / loop_bytes;
+	  do_ifelse (CCmode, EQ, NULL_RTX, NULL_RTX, ccreg, final_label,
+		     profile_probability::unlikely ());
+
+	  /* cmp_rem equals to 3 bytes and leave 1 byte to load and
+	     compare.  */
+	  do_add3 (src1_addr, src1_addr, GEN_INT (2));
+	  do_add3 (src2_addr, src2_addr, GEN_INT (2));
 	}
-      emit_move_insn (iter, GEN_INT (niter));
-      emit_move_insn (loop_cmp, GEN_INT (niter * loop_bytes));
-      emit_move_insn (cmp_rem, GEN_INT (bytes - niter * loop_bytes));
-    }
-  else
-    {
-      library_call_label = gen_label_rtx ();

-      /* If we go to the cleanup code, it expects length to be in cmp_rem.  */
-      emit_move_insn (cmp_rem, bytes_rtx);
+      emit_label (cmp_1);
+    }

-      /* Check for > max_bytes bytes.  We want to bail out as quickly as
-	 possible if we have to go over to memcmp.  */
-      do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (max_bytes),
-		 NULL_RTX, library_call_label, profile_probability::even ());
+  /* Do 1 byte load and compare.  */
+  do_load_and_compare (QImode, src1_addr, src2_addr, dcond, diff,
+		       orig_src1, orig_src2);
+  do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+	     *dcond, diff_label, profile_probability::likely ());
+  j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
+  JUMP_LABEL (j) = final_label;
+  LABEL_NUSES (final_label) += 1;
+  emit_barrier ();
+}

-      /* Check for < loop_bytes bytes.  */
-      do_ifelse (CCmode, LT, bytes_rtx, GEN_INT (loop_bytes),
-		 NULL_RTX, cleanup_label, profile_probability::even ());
+/* Generate code to convert a DImode-plus-carry subtract result into
+   a SImode result that has the same <0 / ==0 / >0 properties to
+   produce the final result from memcmp.

-      /* Loop compare bytes and iterations if bytes>max_bytes.  */
-      rtx mb_reg = gen_reg_rtx (word_mode);
-      emit_move_insn (mb_reg, GEN_INT (max_loop_bytes));
-      rtx mi_reg = gen_reg_rtx (word_mode);
-      emit_move_insn (mi_reg, GEN_INT (max_loop_iter));
+   TARGET is the rtx for the register to receive the memcmp result.
+   SUB_RESULT is the rtx for the register contining the subtract result.  */

-      /* Compute number of loop iterations if bytes <= max_bytes.  */
-      if (word_mode == DImode)
-	emit_insn (gen_lshrdi3 (iter, bytes_rtx, GEN_INT (l2lb)));
-      else
-	emit_insn (gen_lshrsi3 (iter, bytes_rtx, GEN_INT (l2lb)));
+static void
+generate_6432_conversion (rtx target, rtx sub_result)
+{
+  /* We need to produce DI result from sub, then convert to target SI
+     while maintaining <0 / ==0 / >0 properties.  This sequence works:
+     subfc L,A,B
+     subfe H,H,H
+     popcntd L,L
+     rldimi L,H,6,0

-      /* Compute bytes to compare in loop if bytes <= max_bytes.  */
-      rtx mask = GEN_INT (HOST_WIDE_INT_M1U << l2lb);
-      if (word_mode == DImode)
-	{
-	  emit_insn (gen_anddi3 (loop_cmp, bytes_rtx, mask));
-	}
-      else
-	{
-	  emit_insn (gen_andsi3 (loop_cmp, bytes_rtx, mask));
-	}
+     This is an alternate one Segher cooked up if somebody
+     wants to expand this for something that doesn't have popcntd:
+     subfc L,a,b
+     subfe H,x,x
+     addic t,L,-1
+     subfe v,t,L
+     or z,v,H

-      /* Check for bytes <= max_bytes.  */
-      if (TARGET_ISEL)
-	{
-	  /* P9 has fast isel so we use one compare and two isel.  */
-	  cr = gen_reg_rtx (CCmode);
-	  rtx compare_rtx = gen_rtx_COMPARE (CCmode, bytes_rtx,
-					     GEN_INT (max_bytes));
-	  emit_move_insn (cr, compare_rtx);
-	  rtx cmp_rtx = gen_rtx_LE (VOIDmode, cr, const0_rtx);
-	  do_isel (loop_cmp, cmp_rtx, loop_cmp, mb_reg, cr);
-	  do_isel (iter, cmp_rtx, iter, mi_reg, cr);
-	}
-      else
-	{
-	  rtx lab_after = gen_label_rtx ();
-	  do_ifelse (CCmode, LE, bytes_rtx, GEN_INT (max_bytes),
-		     NULL_RTX, lab_after, profile_probability::even ());
-	  emit_move_insn (loop_cmp, mb_reg);
-	  emit_move_insn (iter, mi_reg);
-	  emit_label (lab_after);
-	}
+     And finally, p9 can just do this:
+     cmpld A,B
+     setb r
+     .  */

-      /* Now compute remainder bytes which isn't used until after the loop.  */
-      do_sub3 (cmp_rem, bytes_rtx, loop_cmp);
+  if (TARGET_64BIT)
+    {
+      rtx tmp_reg_ca = gen_reg_rtx (DImode);
+      emit_insn (gen_subfdi3_carry_in_xx (tmp_reg_ca));
+      rtx popcnt = gen_reg_rtx (DImode);
+      emit_insn (gen_popcntddi2 (popcnt, sub_result));
+      rtx tmp2 = gen_reg_rtx (DImode);
+      emit_insn (gen_iordi3 (tmp2, popcnt, tmp_reg_ca));
+      emit_insn (gen_movsi (target, gen_lowpart (SImode, tmp2)));
+    }
+  else
+    {
+      rtx tmp_reg_ca = gen_reg_rtx (SImode);
+      emit_insn (gen_subfsi3_carry_in_xx (tmp_reg_ca));
+      rtx popcnt = gen_reg_rtx (SImode);
+      emit_insn (gen_popcntdsi2 (popcnt, sub_result));
+      emit_insn (gen_iorsi3 (target, popcnt, tmp_reg_ca));
     }
+}

-  rtx dcond = NULL_RTX; /* Used for when we jump to diff_label.  */
-  /* For p9 we need to have just one of these as multiple places define
-     it and it gets used by the setb at the end.  */
+/* Generate the return value when memcmp finds a difference from
+   compare.  */
+static void
+gen_diff_handle (rtx target, rtx dcond, rtx diff, rtx diff_label,
+		 rtx final_label)
+{
+  emit_label (diff_label);
   if (TARGET_P9_MISC)
-    dcond = gen_reg_rtx (CCUNSmode);
+    emit_insn (gen_setb_unsigned (target, dcond));
+  else
+    generate_6432_conversion (target, diff);

-  if (!bytes_is_const || bytes >= loop_bytes)
-    {
-      /* It should not be possible to come here if remaining bytes is
-	 < 16 in the runtime case either.  Compute number of loop
-	 iterations.  We compare 2*word_mode per iteration so 16B for
-	 64-bit code and 8B for 32-bit.  Set up two induction
-	 variables and load count register.  */
-
-      /* HACK ALERT: create hard reg for CTR here.  If we just use a
-	 pseudo, cse will get rid of it and then the allocator will
-	 see it used in the lshr above and won't give us ctr.  */
-      rtx ctr = gen_rtx_REG (Pmode, CTR_REGNO);
-      emit_move_insn (ctr, iter);
-      emit_move_insn (diff, GEN_INT (0));
-      emit_move_insn (iv1, GEN_INT (0));
-      emit_move_insn (iv2, GEN_INT (load_mode_size));
-
-      /* inner loop to compare 2*word_mode */
-      rtx loop_top_label = gen_label_rtx ();
-      emit_label (loop_top_label);
-
-      rtx src1_ix1 = gen_rtx_PLUS (word_mode, src1_addr, iv1);
-      rtx src2_ix1 = gen_rtx_PLUS (word_mode, src2_addr, iv1);
-
-      do_load_for_compare_from_addr (load_mode, d1_1,
-				     src1_ix1, orig_src1);
-      do_load_for_compare_from_addr (load_mode, d2_1,
-				     src2_ix1, orig_src2);
-      do_add3 (iv1, iv1, GEN_INT (loop_bytes));
-
-      rtx src1_ix2 = gen_rtx_PLUS (word_mode, src1_addr, iv2);
-      rtx src2_ix2 = gen_rtx_PLUS (word_mode, src2_addr, iv2);
-
-      do_load_for_compare_from_addr (load_mode, d1_2,
-				     src1_ix2, orig_src1);
-      do_load_for_compare_from_addr (load_mode, d2_2,
-				     src2_ix2, orig_src2);
-      do_add3 (iv2, iv2, GEN_INT (loop_bytes));
-
-      if (TARGET_P9_MISC)
-	{
-	  /* Generate a compare, and convert with a setb later.  */
-	  rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1_1, d2_1);
-	  emit_insn (gen_rtx_SET (dcond, cmp));
-	}
-      else
-	{
-	  dcond = gen_reg_rtx (CCmode);
-	  if (word_mode == DImode)
-	    emit_insn (gen_subfdi3_carry_dot2 (diff, d2_1, d1_1, dcond));
-	  else
-	    emit_insn (gen_subfsi3_carry_dot2 (diff, d2_1, d1_1, dcond));
-	}
+  rtx final_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
+  rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
+  JUMP_LABEL (j) = final_label;
+  LABEL_NUSES (final_label) += 1;
+  emit_barrier ();
+}

-      do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
-		 dcond, diff_label, profile_probability::unlikely ());
+static void
+gen_load_compare_loop (machine_mode load_mode, rtx src1_addr, rtx src2_addr,
+		       rtx orig_src1, rtx orig_src2, rtx diff, rtx diff_label,
+		       rtx cmp_rem, rtx *dcond, HOST_WIDE_INT loop_bytes,
+		       rtx final_label, bool bytes_is_const)
+{
+  HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
+  gcc_assert (loop_bytes == 2 * load_mode_size);

-      if (TARGET_P9_MISC)
-	{
-	  /* Generate a compare, and convert with a setb later.  */
-	  rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1_2, d2_2);
-	  emit_insn (gen_rtx_SET (dcond, cmp));
-	}
-      else
-	{
-	  dcond = gen_reg_rtx (CCmode);
-	  if (word_mode == DImode)
-	    emit_insn (gen_subfdi3_carry_dot2 (diff, d2_2, d1_2, dcond));
-	  else
-	    emit_insn (gen_subfsi3_carry_dot2 (diff, d2_2, d1_2, dcond));
-	}
+  rtx iter = gen_reg_rtx (word_mode);

-      rtx eqrtx = gen_rtx_EQ (VOIDmode, d1_2, d2_2);
-      if (TARGET_64BIT)
-	j = emit_jump_insn (gen_bdnztf_di (loop_top_label, ctr, ctr,
-					   eqrtx, dcond));
-      else
-	j = emit_jump_insn (gen_bdnztf_si (loop_top_label, ctr, ctr,
-					   eqrtx, dcond));
-      add_reg_br_prob_note (j, profile_probability::likely ());
-      JUMP_LABEL (j) = loop_top_label;
-      LABEL_NUSES (loop_top_label) += 1;
-    }
+  int l2lb = floor_log2 (loop_bytes);
+  if (word_mode == DImode)
+    emit_insn (gen_lshrdi3 (iter, cmp_rem, GEN_INT (l2lb)));
+  else
+    emit_insn (gen_lshrsi3 (iter, cmp_rem, GEN_INT (l2lb)));

-  HOST_WIDE_INT bytes_remaining = 0;
-  if (bytes_is_const)
-    bytes_remaining = (bytes % loop_bytes);
+  rtx ctr = gen_rtx_REG (Pmode, CTR_REGNO);
+  emit_move_insn (ctr, iter);

-  /* If diff is nonzero, branch to difference handling
-     code.  If we exit here with a nonzero diff, it is
-     because the second word differed.  */
-  if (TARGET_P9_MISC)
-    do_ifelse (CCUNSmode, NE, NULL_RTX, NULL_RTX, dcond,
-	       diff_label, profile_probability::unlikely ());
+  rtx iv1 = gen_reg_rtx (word_mode);
+  rtx iv2 = gen_reg_rtx (word_mode);
+  rtx d1_1 = gen_reg_rtx (word_mode);  /* Addr expression src1+iv1.  */
+  rtx d1_2 = gen_reg_rtx (word_mode);  /* Addr expression src1+iv2.  */
+  rtx d2_1 = gen_reg_rtx (word_mode);  /* Addr expression src2+iv1.  */
+  rtx d2_2 = gen_reg_rtx (word_mode);  /* Addr expression src2+iv2.  */
+
+  emit_move_insn (iv1, GEN_INT (0));
+  emit_move_insn (iv2, GEN_INT (load_mode_size));
+
+  rtx loop_top_label = gen_label_rtx ();
+  emit_label (loop_top_label);
+
+  /* Manually put two pair of loads together.  */
+  rtx src1_ix1 = gen_rtx_PLUS (word_mode, src1_addr, iv1);
+  rtx src2_ix1 = gen_rtx_PLUS (word_mode, src2_addr, iv1);
+  do_load_for_compare_from_addr (load_mode, d1_1, src1_ix1, orig_src1);
+  do_load_for_compare_from_addr (load_mode, d2_1, src2_ix1, orig_src2);
+  rtx src1_ix2 = gen_rtx_PLUS (word_mode, src1_addr, iv2);
+  rtx src2_ix2 = gen_rtx_PLUS (word_mode, src2_addr, iv2);
+  do_load_for_compare_from_addr (load_mode, d1_2, src1_ix2, orig_src1);
+  do_load_for_compare_from_addr (load_mode, d2_2, src2_ix2, orig_src2);
+
+  do_reg_compare (false, NULL_RTX, diff, dcond, d1_1, d2_1);
+  do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX,
+	     *dcond, diff_label, profile_probability::unlikely ());
+  do_reg_compare (false, NULL_RTX, diff, dcond, d1_2, d2_2);
+
+  do_add3 (iv1, iv1, GEN_INT (loop_bytes));
+  do_add3 (iv2, iv2, GEN_INT (loop_bytes));
+
+  rtx eqrtx = gen_rtx_EQ (VOIDmode, d1_2, d2_2);
+  rtx_insn *j;
+  if (TARGET_64BIT)
+    j = emit_jump_insn (gen_bdnztf_di (loop_top_label, ctr, ctr,
+				       eqrtx, *dcond));
   else
-    do_ifelse (CCmode, NE, diff, const0_rtx, NULL_RTX,
-	       diff_label, profile_probability::unlikely ());
+    j = emit_jump_insn (gen_bdnztf_si (loop_top_label, ctr, ctr,
+				       eqrtx, *dcond));
+  add_reg_br_prob_note (j, profile_probability::likely ());
+  JUMP_LABEL (j) = loop_top_label;
+  LABEL_NUSES (loop_top_label) += 1;

-  if (library_call_label != NULL && bytes_is_const && bytes > max_bytes)
-    {
-      /* If the length is known at compile time, then we will always
-	 have a remainder to go to the library call with.  */
-      rtx library_call_ref = gen_rtx_LABEL_REF (VOIDmode, library_call_label);
-      j = emit_jump_insn (gen_rtx_SET (pc_rtx, library_call_ref));
-      JUMP_LABEL (j) = library_call_label;
-      LABEL_NUSES (library_call_label) += 1;
-      emit_barrier ();
-    }
+  do_ifelse (GET_MODE (*dcond), NE, NULL_RTX, NULL_RTX, *dcond,
+	     diff_label, profile_probability::unlikely ());

-  if (bytes_is_const && bytes_remaining == 0)
+  /* If length is fixed, we know how many bytes are left.  So skip the
+     remain bytes test.  */
+  if (!bytes_is_const)
     {
-      /* No remainder and if we are here then diff is 0 so just return 0 */
-      if (TARGET_64BIT)
-	emit_insn (gen_movsi (target, gen_lowpart (SImode, diff)));
-      else
-	emit_move_insn (target, diff);
-      j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
-      JUMP_LABEL (j) = final_label;
-      LABEL_NUSES (final_label) += 1;
-      emit_barrier ();
+      do_sub3 (cmp_rem, cmp_rem, iv1);
+      do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, NULL_RTX,
+		 final_label, profile_probability::unlikely ());
     }
-  else if (!no_remainder_code)
-    {
-      /* Update addresses to point to the next word to examine.  */
-      do_add3 (src1_addr, src1_addr, iv1);
-      do_add3 (src2_addr, src2_addr, iv1);
-
-      emit_label (cleanup_label);

-      if (!bytes_is_const)
-	{
-	  /* If we're dealing with runtime length, we have to check if
-	     it's zero after the loop.  When length is known at compile
-	     time the no-remainder condition is dealt with above.  By
-	     doing this after cleanup_label, we also deal with the
-	     case where length is 0 at the start and we bypass the
-	     loop with a branch to cleanup_label.  */
-	  emit_move_insn (target, const0_rtx);
-	  do_ifelse (CCmode, EQ, cmp_rem, const0_rtx,
-		     NULL_RTX, final_label, profile_probability::unlikely ());
-	}
-
-      rtx final_cleanup = gen_label_rtx ();
-      rtx cmp_rem_before = gen_reg_rtx (word_mode);
-      /* Compare one more word_mode chunk if needed.  */
-      if (!bytes_is_const || bytes_remaining >= load_mode_size)
-	{
-	  /* If remainder length < word length, branch to final
-	     cleanup compare.  */
-	
-	  if (!bytes_is_const)
-	    {
-	      do_ifelse (CCmode, LT, cmp_rem, GEN_INT (load_mode_size),
-			 NULL_RTX, final_cleanup, profile_probability::even ());
-	    }
+  do_add3 (src1_addr, src1_addr, iv1);
+  do_add3 (src2_addr, src2_addr, iv1);
+}

-	  /* load and compare 8B */
-	  do_load_for_compare_from_addr (load_mode, d1_1,
-					 src1_addr, orig_src1);
-	  do_load_for_compare_from_addr (load_mode, d2_1,
-					 src2_addr, orig_src2);
+/* Generate memcmp library call.  */
+static void
+gen_library_call (rtx target, rtx src1_addr, rtx src2_addr, rtx bytes_rtx,
+		  rtx library_label)
+{
+  emit_label (library_label);
+
+  rtx len_rtx = gen_reg_rtx (word_mode);
+  emit_move_insn (len_rtx, bytes_rtx);
+  tree fun = builtin_decl_explicit (BUILT_IN_MEMCMP);
+  emit_library_call_value (XEXP (DECL_RTL (fun), 0),
+			   target, LCT_NORMAL, GET_MODE (target),
+			   src1_addr, Pmode, src2_addr, Pmode,
+			   len_rtx, GET_MODE (len_rtx));
+}

-	  /* Compare the word, see if we need to do the last partial.  */
-	  if (TARGET_P9_MISC)
-	    {
-	      /* Generate a compare, and convert with a setb later.  */
-	      rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1_1, d2_1);
-	      emit_insn (gen_rtx_SET (dcond, cmp));
-	    }
-	  else
-	    {
-	      dcond = gen_reg_rtx (CCmode);
-	      if (word_mode == DImode)
-		emit_insn (gen_subfdi3_carry_dot2 (diff, d2_1, d1_1, dcond));
-	      else
-		emit_insn (gen_subfsi3_carry_dot2 (diff, d2_1, d1_1, dcond));
-	    }
+static bool
+expand_compare_with_fixed_length (rtx operands[])
+{
+  rtx target = operands[0];
+  rtx orig_src1 = operands[1];
+  rtx orig_src2 = operands[2];
+  rtx bytes_rtx = operands[3];
+  rtx align_rtx = operands[4];

-	  do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
-		     dcond, diff_label, profile_probability::even ());
+  gcc_assert (CONST_INT_P (bytes_rtx));
+  gcc_assert (GET_MODE (target) == SImode);

-	  do_add3 (src1_addr, src1_addr, GEN_INT (load_mode_size));
-	  do_add3 (src2_addr, src2_addr, GEN_INT (load_mode_size));
-	  emit_move_insn (cmp_rem_before, cmp_rem);
-	  do_add3 (cmp_rem, cmp_rem, GEN_INT (-load_mode_size));
-	  if (bytes_is_const)
-	    bytes_remaining -= load_mode_size;
-	  else
-	    /* See if remaining length is now zero.  We previously set
-	       target to 0 so we can just jump to the end.  */
-	    do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, NULL_RTX,
-		       final_label, profile_probability::unlikely ());
-	}
+  if (TARGET_32BIT && TARGET_POWERPC64)
+    return false;

-      /* Cases:
-	 bytes_is_const
-	   We can always shift back to do an overlapping compare
-	   of the last chunk because we know length >= 8.
-
-	 !bytes_is_const
-	   align>=load_mode_size
-	     Read word_mode and mask
-	   align<load_mode_size
-	     avoid stepping past end
-
-	  Three strategies:
-	  * decrement address and do overlapping compare
-	  * read word_mode and mask
-	  * carefully avoid crossing 4k boundary
-       */
-
-      if ((!bytes_is_const || (bytes_is_const && bytes_remaining && isP7))
-	  && align1 >= load_mode_size && align2 >= load_mode_size)
-	{
-	  /* Alignment is larger than word_mode so we do not need to be
-	     concerned with extra page crossings.  But, we do not know
-	     that the length is larger than load_mode_size so we might
-	     end up compareing against data before the block if we try
-	     an overlapping compare.  Also we use this on P7 for fixed length
-	     remainder because P7 doesn't like overlapping unaligned.
-	     Strategy: load 8B, shift off bytes past length, and compare.  */
-	  emit_label (final_cleanup);
-	  do_load_mask_compare (load_mode, diff, cmp_rem, dcond,
-				src1_addr, src2_addr, orig_src1, orig_src2);
-	}
-      else if (bytes_remaining && bytes_is_const)
-	{
-	  /* We do not do loop expand if length < 32 so we know at the
-	     end we can do an overlapping compare.
-	     Strategy: shift address back and do word_mode load that
-	     ends at the end of the block.  */
-	  emit_label (final_cleanup);
-	  do_overlap_load_compare (load_mode, true, bytes_remaining, diff,
-				   cmp_rem, dcond, src1_addr, src2_addr,
-				   orig_src1, orig_src2);
-	}
-      else if (!bytes_is_const)
-	{
-	  rtx handle4k_label = gen_label_rtx ();
-	  rtx nonconst_overlap = gen_label_rtx ();
-	  emit_label (nonconst_overlap);
-
-	  /* Here we have to handle the case where whe have runtime
-	     length which may be too short for overlap compare, and
-	     alignment is not at least load_mode_size so we have to
-	     tread carefully to avoid stepping across 4k boundaries.  */
-
-	  /* If the length after the loop was larger than word_mode
-	     size, we can just do an overlapping compare and we're
-	     done.  We fall through to this code from the word_mode
-	     compare that preceeds this.  */
-	  do_overlap_load_compare (load_mode, false, 0, diff,
-				   cmp_rem, dcond, src1_addr, src2_addr,
-				   orig_src1, orig_src2);
-
-	  rtx diff_ref = gen_rtx_LABEL_REF (VOIDmode, diff_label);
-	  j = emit_jump_insn (gen_rtx_SET (pc_rtx, diff_ref));
-	  JUMP_LABEL (j) = diff_label;
-	  LABEL_NUSES (diff_label) += 1;
-	  emit_barrier ();
-
-	  /* If we couldn't do the overlap compare we have to be more
-	     careful of the 4k boundary.  Test to see if either
-	     address is less than word_mode_size away from a 4k
-	     boundary.  If not, then we can do a load/shift/compare
-	     and we are done.  We come to this code if length was less
-	     than word_mode_size.  */
-
-	  emit_label (final_cleanup);
-
-	  /* We can still avoid the slow case if the length was larger
-	     than one loop iteration, in which case go do the overlap
-	     load compare path.  */
-	  do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (loop_bytes),
-		     NULL_RTX, nonconst_overlap, profile_probability::even ());
-
-	  rtx rem4k = gen_reg_rtx (word_mode);
-	  rtx dist1 = gen_reg_rtx (word_mode);
-	  rtx dist2 = gen_reg_rtx (word_mode);
-	  do_sub3 (rem4k, GEN_INT (4096), cmp_rem);
-	  if (word_mode == SImode)
-	    emit_insn (gen_andsi3 (dist1, src1_addr, GEN_INT (0xfff)));
-	  else
-	    emit_insn (gen_anddi3 (dist1, src1_addr, GEN_INT (0xfff)));
-	  do_ifelse (CCmode, LE, dist1, rem4k, NULL_RTX,
-		     handle4k_label, profile_probability::very_unlikely ());
-	  if (word_mode == SImode)
-	    emit_insn (gen_andsi3 (dist2, src2_addr, GEN_INT (0xfff)));
-	  else
-	    emit_insn (gen_anddi3 (dist2, src2_addr, GEN_INT (0xfff)));
-	  do_ifelse (CCmode, LE, dist2, rem4k, NULL_RTX,
-		     handle4k_label, profile_probability::very_unlikely ());
+  /* This must be a fixed size alignment.  */
+  if (!CONST_INT_P (align_rtx))
+    return false;

-	  /* We don't have a 4k boundary to deal with, so do
-	     a load/shift/compare and jump to diff.  */
+  HOST_WIDE_INT align = INTVAL (align_rtx) / BITS_PER_UNIT;
+  HOST_WIDE_INT bytes = INTVAL (bytes_rtx);

-	  do_load_mask_compare (load_mode, diff, cmp_rem, dcond,
-				src1_addr, src2_addr, orig_src1, orig_src2);
+  if (bytes == 0)
+    return true;

-	  j = emit_jump_insn (gen_rtx_SET (pc_rtx, diff_ref));
-	  JUMP_LABEL (j) = diff_label;
-	  LABEL_NUSES (diff_label) += 1;
-	  emit_barrier ();
+  /* Limit the amount we compare, if known statically.  */
+  HOST_WIDE_INT max_bytes = get_max_inline_loop_bytes (true, align);

-	  /* Finally in the unlikely case we are inching up to a
-	     4k boundary we use a compact lbzx/compare loop to do
-	     it a byte at a time.  */
+    /* Allow the option to override the default.  */
+  if (rs6000_block_compare_inline_loop_limit >= 0)
+    max_bytes = (unsigned HOST_WIDE_INT) rs6000_block_compare_inline_loop_limit;

-	  emit_label (handle4k_label);
+  if (max_bytes == 0)
+    return false;

-	  rtx ctr = gen_rtx_REG (Pmode, CTR_REGNO);
-	  emit_move_insn (ctr, cmp_rem);
-	  rtx ixreg = gen_reg_rtx (Pmode);
-	  emit_move_insn (ixreg, const0_rtx);
+  machine_mode load_mode = word_mode;
+  HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);

-	  rtx src1_ix = gen_rtx_PLUS (word_mode, src1_addr, ixreg);
-	  rtx src2_ix = gen_rtx_PLUS (word_mode, src2_addr, ixreg);
-	  rtx d1 = gen_reg_rtx (word_mode);
-	  rtx d2 = gen_reg_rtx (word_mode);
+  if (max_bytes < load_mode_size
+      || !IN_RANGE (bytes, load_mode_size, max_bytes))
+   return false;

-	  rtx fc_loop = gen_label_rtx ();
-	  emit_label (fc_loop);
+  /* Remainder bytes for compare.  */
+  rtx cmp_rem = gen_reg_rtx (word_mode);
+  /* Number of bytes per iteration of the unrolled loop.  */
+  HOST_WIDE_INT loop_bytes = 2 * load_mode_size;

-	  do_load_for_compare_from_addr (QImode, d1, src1_ix, orig_src1);
-	  do_load_for_compare_from_addr (QImode, d2, src2_ix, orig_src2);
+  rtx src1_addr = copy_addr_to_reg (XEXP (orig_src1, 0));
+  rtx src2_addr = copy_addr_to_reg (XEXP (orig_src2, 0));

-	  do_add3 (ixreg, ixreg, const1_rtx);
+  /* Label for set target when finding a diff.  */
+  rtx diff_label = gen_label_rtx ();
+  rtx final_label = gen_label_rtx ();

-	  rtx cond = gen_reg_rtx (CCmode);
-	  rtx subexpr = gen_rtx_MINUS (word_mode, d1, d2);
-	  rs6000_emit_dot_insn (diff, subexpr, 2, cond);
+  /* CC used for when we jump to diff_label.  */
+  rtx dcond = NULL_RTX;
+  /* For p9 we need to have just one of these as multiple places define
+     it and it gets used by the setb at the end.  */
+  if (TARGET_P9_MISC)
+    dcond = gen_reg_rtx (CCUNSmode);

-	  rtx eqrtx = gen_rtx_EQ (VOIDmode, d1, d2);
-	  if (TARGET_64BIT)
-	    j = emit_jump_insn (gen_bdnztf_di (fc_loop, ctr, ctr,
-					       eqrtx, cond));
-	  else
-	    j = emit_jump_insn (gen_bdnztf_si (fc_loop, ctr, ctr,
-					       eqrtx, cond));
-	  add_reg_br_prob_note (j, profile_probability::likely ());
-	  JUMP_LABEL (j) = fc_loop;
-	  LABEL_NUSES (fc_loop) += 1;
-
-	  if (TARGET_64BIT)
-	    emit_insn (gen_movsi (target, gen_lowpart (SImode, diff)));
-	  else
-	    emit_move_insn (target, diff);
-
-	  /* Since we are comparing bytes, the difference can be used
-	     as the final result and we are done here.  */
-	  j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
-	  JUMP_LABEL (j) = final_label;
-	  LABEL_NUSES (final_label) += 1;
-	  emit_barrier ();
-	}
-    }
+  /* Difference found is stored here before jump to diff_label.  */
+  rtx diff = gen_reg_rtx (word_mode);

-  emit_label (diff_label);
-  /* difference handling, 64->32 conversion */
+  emit_move_insn (cmp_rem, GEN_INT (bytes));
+  emit_move_insn (target, const0_rtx);

-  /* We need to produce DI result from sub, then convert to target SI
-     while maintaining <0 / ==0 / >0 properties.  This sequence works:
-     subfc L,A,B
-     subfe H,H,H
-     popcntd L,L
-     rldimi L,H,6,0
+  gen_load_compare_loop (load_mode, src1_addr, src2_addr, orig_src1,
+			 orig_src2, diff, diff_label, cmp_rem, &dcond,
+			 loop_bytes, final_label, true);

-     This is an alternate one Segher cooked up if somebody
-     wants to expand this for something that doesn't have popcntd:
-     subfc L,a,b
-     subfe H,x,x
-     addic t,L,-1
-     subfe v,t,L
-     or z,v,H
+  HOST_WIDE_INT rem_bytes = bytes % loop_bytes;

-     And finally, p9 can just do this:
-     cmpld A,B
-     setb r */
-
-  if (TARGET_P9_MISC)
-    emit_insn (gen_setb_unsigned (target, dcond));
-  else
+  if (rem_bytes >= load_mode_size)
     {
-      if (TARGET_64BIT)
-	{
-	  rtx tmp_reg_ca = gen_reg_rtx (DImode);
-	  emit_insn (gen_subfdi3_carry_in_xx (tmp_reg_ca));
-	  emit_insn (gen_popcntddi2 (diff, diff));
-	  emit_insn (gen_iordi3 (diff, diff, tmp_reg_ca));
-	  emit_insn (gen_movsi (target, gen_lowpart (SImode, diff)));
-	}
-      else
-	{
-	  rtx tmp_reg_ca = gen_reg_rtx (SImode);
-	  emit_insn (gen_subfsi3_carry_in_xx (tmp_reg_ca));
-	  emit_insn (gen_popcntdsi2 (diff, diff));
-	  emit_insn (gen_iorsi3 (target, diff, tmp_reg_ca));
-	}
+      do_load_and_compare (load_mode, src1_addr, src2_addr, &dcond, diff,
+			   orig_src1, orig_src2);
+      do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
+		 dcond, diff_label, profile_probability::unlikely ());
+      rem_bytes -= load_mode_size;
     }

-  if (library_call_label != NULL)
+  if (rem_bytes > 0)
     {
-      /* Branch around memcmp call.  */
-      j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
-      JUMP_LABEL (j) = final_label;
-      LABEL_NUSES (final_label) += 1;
-      emit_barrier ();
-
-      /* Make memcmp library call.  cmp_rem is the remaining bytes that
-	 were compared and cmp_rem is the expected amount to be compared
-	 by memcmp.  If we don't find a difference in the loop compare, do
-	 the library call directly instead of doing a small compare just
-	 to get to an arbitrary boundary before calling it anyway.
-	 Also, update addresses to point to the next word to examine.  */
-      emit_label (library_call_label);
-
-      rtx len_rtx = gen_reg_rtx (word_mode);
-      if (bytes_is_const)
-	{
-	  emit_move_insn (len_rtx, cmp_rem);
-	  do_add3 (src1_addr, src1_addr, iv1);
-	  do_add3 (src2_addr, src2_addr, iv1);
-	}
-      else
-	emit_move_insn (len_rtx, bytes_rtx);
-
-      tree fun = builtin_decl_explicit (BUILT_IN_MEMCMP);
-      emit_library_call_value (XEXP (DECL_RTL (fun), 0),
-			       target, LCT_NORMAL, GET_MODE (target),
-			       src1_addr, Pmode,
-			       src2_addr, Pmode,
-			       len_rtx, GET_MODE (len_rtx));
+      do_overlap_load_compare (load_mode, rem_bytes, diff, &dcond,
+			       orig_src1, orig_src2, bytes);
+      do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
+		 dcond, diff_label, profile_probability::unlikely ());
     }

-  /* emit final_label */
+  rtx final_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
+  rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
+  JUMP_LABEL (j) = final_label;
+  LABEL_NUSES (final_label) += 1;
+  emit_barrier ();
+
+  gen_diff_handle (target, dcond, diff, diff_label, final_label);
   emit_label (final_label);
+
   return true;
 }

-/* Generate code to convert a DImode-plus-carry subtract result into
-   a SImode result that has the same <0 / ==0 / >0 properties to
-   produce the final result from memcmp.
+static bool
+expand_compare_with_variable_length (rtx operands[])
+{
+  rtx target = operands[0];
+  rtx orig_src1 = operands[1];
+  rtx orig_src2 = operands[2];
+  rtx bytes_rtx = operands[3];
+  rtx align_rtx = operands[4];

-   TARGET is the rtx for the register to receive the memcmp result.
-   SUB_RESULT is the rtx for the register contining the subtract result.  */
+  gcc_assert (!CONST_INT_P (bytes_rtx));

-void
-generate_6432_conversion(rtx target, rtx sub_result)
-{
-  /* We need to produce DI result from sub, then convert to target SI
-     while maintaining <0 / ==0 / >0 properties.  This sequence works:
-     subfc L,A,B
-     subfe H,H,H
-     popcntd L,L
-     rldimi L,H,6,0
+  if (TARGET_32BIT && TARGET_POWERPC64)
+    return false;

-     This is an alternate one Segher cooked up if somebody
-     wants to expand this for something that doesn't have popcntd:
-     subfc L,a,b
-     subfe H,x,x
-     addic t,L,-1
-     subfe v,t,L
-     or z,v,H
+  /* This must be a fixed size alignment.  */
+  if (!CONST_INT_P (align_rtx))
+    return false;

-     And finally, p9 can just do this:
-     cmpld A,B
-     setb r */
+  HOST_WIDE_INT align = INTVAL (align_rtx) / BITS_PER_UNIT;

-  if (TARGET_64BIT)
+  /* Limit the amount we compare, if known statically.  */
+  HOST_WIDE_INT max_bytes = get_max_inline_loop_bytes (false, align);
+
+  /* Allow the option to override the default.  */
+  if (rs6000_block_compare_inline_loop_limit >= 0)
+    max_bytes = rs6000_block_compare_inline_loop_limit;
+  if (max_bytes == 0)
+    return false;
+
+  /* Remainder bytes for compare.  */
+  rtx cmp_rem = gen_reg_rtx (word_mode);
+
+  /* Strip unneeded subreg from length if there is one.  */
+  if (SUBREG_P (bytes_rtx) && subreg_lowpart_p (bytes_rtx))
+    bytes_rtx = SUBREG_REG (bytes_rtx);
+
+  if (GET_MODE_SIZE (GET_MODE (bytes_rtx)) > GET_MODE_SIZE (word_mode))
+    /* Do not expect length longer than word_mode.  */
+    return false;
+  else if (GET_MODE_SIZE (GET_MODE (bytes_rtx)) < GET_MODE_SIZE (word_mode))
     {
-      rtx tmp_reg_ca = gen_reg_rtx (DImode);
-      emit_insn (gen_subfdi3_carry_in_xx (tmp_reg_ca));
-      rtx popcnt = gen_reg_rtx (DImode);
-      emit_insn (gen_popcntddi2 (popcnt, sub_result));
-      rtx tmp2 = gen_reg_rtx (DImode);
-      emit_insn (gen_iordi3 (tmp2, popcnt, tmp_reg_ca));
-      emit_insn (gen_movsi (target, gen_lowpart (SImode, tmp2)));
+      bytes_rtx = force_reg (GET_MODE (bytes_rtx), bytes_rtx);
+      bytes_rtx = force_reg (word_mode,
+			     gen_rtx_fmt_e (ZERO_EXTEND, word_mode, bytes_rtx));
     }
   else
+    /* Make sure it's in a register before we get started.  */
+    bytes_rtx = force_reg (GET_MODE (bytes_rtx), bytes_rtx);
+
+  machine_mode load_mode = word_mode;
+  HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
+  rtx src1_addr = copy_addr_to_reg (XEXP (orig_src1, 0));
+  rtx src2_addr = copy_addr_to_reg (XEXP (orig_src2, 0));
+
+  rtx library_call_label = gen_label_rtx ();
+
+  /* Call library if length > max_bytes.  */
+  do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (max_bytes),
+	     NULL_RTX, library_call_label, profile_probability::unlikely ());
+
+  /* Number of bytes per iteration of the unrolled loop.  */
+  HOST_WIDE_INT loop_bytes = 2 * load_mode_size;
+
+  /* Label for set target when finding a diff.  */
+  rtx diff_label = gen_label_rtx ();
+  rtx final_label = gen_label_rtx ();
+
+  /* CC used for when we jump to diff_label.  */
+  rtx dcond = NULL_RTX;
+  /* For p9 we need to have just one of these as multiple places define
+     it and it gets used by the setb at the end.  */
+  if (TARGET_P9_MISC)
+    dcond = gen_reg_rtx (CCUNSmode);
+
+  /* Difference found is stored here before jump to diff_label.  */
+  rtx diff = gen_reg_rtx (word_mode);
+
+  emit_move_insn (target, const0_rtx);
+  emit_move_insn (cmp_rem, bytes_rtx);
+
+  /* Number of bytes to be expanded at rest of loop.  */
+  HOST_WIDE_INT expand_bytes = max_bytes % loop_bytes;
+  if (max_bytes >= loop_bytes)
     {
-      rtx tmp_reg_ca = gen_reg_rtx (SImode);
-      emit_insn (gen_subfsi3_carry_in_xx (tmp_reg_ca));
-      rtx popcnt = gen_reg_rtx (SImode);
-      emit_insn (gen_popcntdsi2 (popcnt, sub_result));
-      emit_insn (gen_iorsi3 (target, popcnt, tmp_reg_ca));
+      rtx compare_rest_label = gen_label_rtx ();
+      do_ifelse (CCmode, LT, cmp_rem, GEN_INT (loop_bytes), NULL_RTX,
+		 compare_rest_label, profile_probability::unlikely ());
+      /* max bytes >= cmp_rem >= loop_bytes.  */
+      gen_load_compare_loop (load_mode, src1_addr, src2_addr, orig_src1,
+			     orig_src2, diff, diff_label, cmp_rem, &dcond,
+			     loop_bytes, final_label, false);
+      emit_label (compare_rest_label);
+      expand_bytes = loop_bytes - 1;
     }
+
+  /* cmp_rem < loop_bytes.  */
+  do_load_compare_rest_of_loop (load_mode, src1_addr, src2_addr, cmp_rem,
+				diff, diff_label, &dcond, final_label,
+				orig_src1, orig_src2, loop_bytes,
+				expand_bytes);
+
+  gen_diff_handle (target, dcond, diff, diff_label, final_label);
+  gen_library_call (target, src1_addr, src2_addr, bytes_rtx,
+		    library_call_label);
+  emit_label (final_label);
+
+  return true;
 }

 /* Generate memcmp expansion using in-line non-loop GPR instructions.
@@ -1975,7 +1702,7 @@ expand_block_compare (rtx operands[])
   /* If this is not a fixed size compare, try generating loop code and
      if that fails just call memcmp.  */
   if (!CONST_INT_P (bytes_rtx))
-    return expand_compare_loop (operands);
+    return expand_compare_with_variable_length (operands);

   /* This must be a fixed size alignment.  */
   if (!CONST_INT_P (align_rtx))
@@ -2016,7 +1743,7 @@ expand_block_compare (rtx operands[])
     max_bytes = ((max_bytes + 1) / 2) - 1;

   if (!IN_RANGE (bytes, 1, max_bytes))
-    return expand_compare_loop (operands);
+    return expand_compare_with_fixed_length (operands);

   rtx final_label = NULL;

@@ -2069,7 +1796,7 @@ expand_block_compare (rtx operands[])
 	  if (TARGET_P9_MISC)
 	    emit_insn (gen_setb_unsigned (target, cond));
 	  else
-	    generate_6432_conversion(target, sub_result);
+	    generate_6432_conversion (target, sub_result);
 	}
     }

diff --git a/gcc/testsuite/gcc.target/powerpc/block-cmp-5.c b/gcc/testsuite/gcc.target/powerpc/block-cmp-5.c
new file mode 100644
index 00000000000..60a38030784
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/block-cmp-5.c
@@ -0,0 +1,11 @@
+/* { dg-do compile } */
+/* { dg-options "-O2 -mblock-compare-inline-loop-limit=1" } */
+/* { dg-final { scan-assembler-not {\mbdnzt\M} } }  */
+
+/* Test that no loop will be generated when the inline loop limit is less
+   than the loop bytes (2 * word_mode_size).  */
+
+int foo (const char* s1, const char* s2, int l)
+{
+  return __builtin_memcmp (s1, s2, l);
+}
diff --git a/gcc/testsuite/gcc.target/powerpc/block-cmp-6.c b/gcc/testsuite/gcc.target/powerpc/block-cmp-6.c
new file mode 100644
index 00000000000..0e03f2af943
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/block-cmp-6.c
@@ -0,0 +1,5 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mblock-compare-inline-loop-limit=4" } */
+/* { dg-timeout-factor 2 } */
+
+#include "../../gcc.dg/memcmp-1.c"
diff --git a/gcc/testsuite/gcc.target/powerpc/block-cmp-7.c b/gcc/testsuite/gcc.target/powerpc/block-cmp-7.c
new file mode 100644
index 00000000000..499f5faee17
--- /dev/null
+++ b/gcc/testsuite/gcc.target/powerpc/block-cmp-7.c
@@ -0,0 +1,5 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mblock-compare-inline-loop-limit=32" } */
+/* { dg-timeout-factor 2 } */
+
+#include "../../gcc.dg/memcmp-1.c"
