diff mbox series

[7/9] vect: Adjust vectorizable_load costing on VMAT_CONTIGUOUS_REVERSE

Message ID e0d83a313ba6b9d6dd33bcbd7981f04ce733cc4e.1686573640.git.linkw@linux.ibm.com
State Accepted
Headers
Series vect: Move costing next to the transform for vect load |

Checks

Context Check Description
snail/gcc-patch-check success Github commit url

Commit Message

Kewen.Lin June 13, 2023, 2:03 a.m. UTC 
  This patch adjusts the cost handling on
VMAT_CONTIGUOUS_REVERSE in function vectorizable_load.  We
don't call function vect_model_load_cost for it any more.

This change makes us not miscount some required vector
permutation as the associated test case shows.

gcc/ChangeLog:

	* tree-vect-stmts.cc (vect_model_load_cost): Assert it won't get
	VMAT_CONTIGUOUS_REVERSE any more.
	(vectorizable_load): Adjust the costing handling on
	VMAT_CONTIGUOUS_REVERSE without calling vect_model_load_cost.

gcc/testsuite/ChangeLog:

	* gcc.dg/vect/costmodel/ppc/costmodel-vect-reversed.c: New test.
---
 .../costmodel/ppc/costmodel-vect-reversed.c   |  22 ++++
 gcc/tree-vect-stmts.cc                        | 109 ++++++++++++------
 2 files changed, 93 insertions(+), 38 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/vect/costmodel/ppc/costmodel-vect-reversed.c
diff mbox series

Patch

diff --git a/gcc/testsuite/gcc.dg/vect/costmodel/ppc/costmodel-vect-reversed.c b/gcc/testsuite/gcc.dg/vect/costmodel/ppc/costmodel-vect-reversed.c
new file mode 100644
index 00000000000..651274be038
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/costmodel/ppc/costmodel-vect-reversed.c
@@ -0,0 +1,22 @@ 
+/* { dg-do compile } */
+/* { dg-require-effective-target vect_int } */
+/* { dg-require-effective-target powerpc_vsx_ok } */
+/* { dg-additional-options "-mvsx" } */
+
+/* Verify we do cost the required vec_perm.  */
+
+int x[1024], y[1024];
+
+void
+foo ()
+{
+  for (int i = 0; i < 512; ++i)
+    {
+      x[2 * i] = y[1023 - (2 * i)];
+      x[2 * i + 1] = y[1023 - (2 * i + 1)];
+    }
+}
+/* The reason why it doesn't check the exact count is that
+   retrying for the epilogue with partial vector capability
+   like Power10 can result in more than 1 vec_perm.  */
+/* { dg-final { scan-tree-dump {\mvec_perm\M} "vect" } } */
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 4c5ce2ab278..7f8d9db5363 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -1134,11 +1134,8 @@  vect_model_load_cost (vec_info *vinfo,
 		      slp_tree slp_node,
 		      stmt_vector_for_cost *cost_vec)
 {
-  gcc_assert (memory_access_type != VMAT_GATHER_SCATTER
-	      && memory_access_type != VMAT_INVARIANT
-	      && memory_access_type != VMAT_ELEMENTWISE
-	      && memory_access_type != VMAT_STRIDED_SLP
-	      && memory_access_type != VMAT_LOAD_STORE_LANES);
+  gcc_assert (memory_access_type == VMAT_CONTIGUOUS
+	      || memory_access_type == VMAT_CONTIGUOUS_PERMUTE);
 
   unsigned int inside_cost = 0, prologue_cost = 0;
   bool grouped_access_p = STMT_VINFO_GROUPED_ACCESS (stmt_info);
@@ -10292,7 +10289,7 @@  vectorizable_load (vec_info *vinfo,
 			      = record_stmt_cost (cost_vec, cnunits,
 						  scalar_load, stmt_info, 0,
 						  vect_body);
-			    goto vec_num_loop_costing_end;
+			    break;
 			  }
 			if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
 			  vec_offset = vec_offsets[vec_num * j + i];
@@ -10335,7 +10332,7 @@  vectorizable_load (vec_info *vinfo,
 			    inside_cost
 			      = record_stmt_cost (cost_vec, 1, vec_construct,
 						  stmt_info, 0, vect_body);
-			    goto vec_num_loop_costing_end;
+			    break;
 			  }
 			unsigned HOST_WIDE_INT const_offset_nunits
 			  = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype)
@@ -10390,7 +10387,7 @@  vectorizable_load (vec_info *vinfo,
 		      }
 
 		    if (costing_p)
-		      goto vec_num_loop_costing_end;
+		      break;
 
 		    align =
 		      known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
@@ -10563,7 +10560,7 @@  vectorizable_load (vec_info *vinfo,
 		case dr_explicit_realign:
 		  {
 		    if (costing_p)
-		      goto vec_num_loop_costing_end;
+		      break;
 		    tree ptr, bump;
 
 		    tree vs = size_int (TYPE_VECTOR_SUBPARTS (vectype));
@@ -10627,7 +10624,7 @@  vectorizable_load (vec_info *vinfo,
 		case dr_explicit_realign_optimized:
 		  {
 		    if (costing_p)
-		      goto vec_num_loop_costing_end;
+		      break;
 		    if (TREE_CODE (dataref_ptr) == SSA_NAME)
 		      new_temp = copy_ssa_name (dataref_ptr);
 		    else
@@ -10650,22 +10647,37 @@  vectorizable_load (vec_info *vinfo,
 		default:
 		  gcc_unreachable ();
 		}
-	      vec_dest = vect_create_destination_var (scalar_dest, vectype);
-	      /* DATA_REF is null if we've already built the statement.  */
-	      if (data_ref)
+
+	      /* One common place to cost the above vect load for different
+		 alignment support schemes.  */
+	      if (costing_p)
 		{
-		  vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
-		  new_stmt = gimple_build_assign (vec_dest, data_ref);
+		  if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
+		    vect_get_load_cost (vinfo, stmt_info, 1,
+					alignment_support_scheme, misalignment,
+					false, &inside_cost, &prologue_cost,
+					cost_vec, cost_vec, true);
+		}
+	      else
+		{
+		  vec_dest = vect_create_destination_var (scalar_dest, vectype);
+		  /* DATA_REF is null if we've already built the statement.  */
+		  if (data_ref)
+		    {
+		      vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
+		      new_stmt = gimple_build_assign (vec_dest, data_ref);
+		    }
+		  new_temp = make_ssa_name (vec_dest, new_stmt);
+		  gimple_set_lhs (new_stmt, new_temp);
+		  vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 		}
-	      new_temp = make_ssa_name (vec_dest, new_stmt);
-	      gimple_set_lhs (new_stmt, new_temp);
-	      vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
 
 	      /* 3. Handle explicit realignment if necessary/supported.
 		 Create in loop:
 		   vec_dest = realign_load (msq, lsq, realignment_token)  */
-	      if (alignment_support_scheme == dr_explicit_realign_optimized
-		  || alignment_support_scheme == dr_explicit_realign)
+	      if (!costing_p
+		  && (alignment_support_scheme == dr_explicit_realign_optimized
+		      || alignment_support_scheme == dr_explicit_realign))
 		{
 		  lsq = gimple_assign_lhs (new_stmt);
 		  if (!realignment_token)
@@ -10690,26 +10702,34 @@  vectorizable_load (vec_info *vinfo,
 
 	      if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
 		{
-		  tree perm_mask = perm_mask_for_reverse (vectype);
-		  new_temp = permute_vec_elements (vinfo, new_temp, new_temp,
-						   perm_mask, stmt_info, gsi);
-		  new_stmt = SSA_NAME_DEF_STMT (new_temp);
+		  if (costing_p)
+		    inside_cost = record_stmt_cost (cost_vec, 1, vec_perm,
+						    stmt_info, 0, vect_body);
+		  else
+		    {
+		      tree perm_mask = perm_mask_for_reverse (vectype);
+		      new_temp
+			= permute_vec_elements (vinfo, new_temp, new_temp,
+						perm_mask, stmt_info, gsi);
+		      new_stmt = SSA_NAME_DEF_STMT (new_temp);
+		    }
 		}
 
 	      /* Collect vector loads and later create their permutation in
 		 vect_transform_grouped_load ().  */
-	      if (grouped_load || slp_perm)
+	      if (!costing_p && (grouped_load || slp_perm))
 		dr_chain.quick_push (new_temp);
 
 	      /* Store vector loads in the corresponding SLP_NODE.  */
-	      if (slp && !slp_perm)
+	      if (!costing_p && slp && !slp_perm)
 		SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt);
 
 	      /* With SLP permutation we load the gaps as well, without
 	         we need to skip the gaps after we manage to fully load
 		 all elements.  group_gap_adj is DR_GROUP_SIZE here.  */
 	      group_elt += nunits;
-	      if (maybe_ne (group_gap_adj, 0U)
+	      if (!costing_p
+		  && maybe_ne (group_gap_adj, 0U)
 		  && !slp_perm
 		  && known_eq (group_elt, group_size - group_gap_adj))
 		{
@@ -10724,8 +10744,6 @@  vectorizable_load (vec_info *vinfo,
 						 gsi, stmt_info, bump);
 		  group_elt = 0;
 		}
-vec_num_loop_costing_end:
-	      ;
 	    }
 	  /* Bump the vector pointer to account for a gap or for excess
 	     elements loaded for a permuted SLP load.  */
@@ -10748,18 +10766,30 @@  vec_num_loop_costing_end:
       if (slp && !slp_perm)
 	continue;
 
-      if (slp_perm && !costing_p)
-        {
+      if (slp_perm)
+	{
 	  unsigned n_perms;
 	  /* For SLP we know we've seen all possible uses of dr_chain so
 	     direct vect_transform_slp_perm_load to DCE the unused parts.
 	     ???  This is a hack to prevent compile-time issues as seen
 	     in PR101120 and friends.  */
-	  bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
-						  gsi, vf, false, &n_perms,
-						  nullptr, true);
-	  gcc_assert (ok);
-        }
+	  if (costing_p
+	      && memory_access_type != VMAT_CONTIGUOUS
+	      && memory_access_type != VMAT_CONTIGUOUS_PERMUTE)
+	    {
+	      vect_transform_slp_perm_load (vinfo, slp_node, vNULL, nullptr, vf,
+					    true, &n_perms, nullptr);
+	      inside_cost = record_stmt_cost (cost_vec, n_perms, vec_perm,
+					      stmt_info, 0, vect_body);
+	    }
+	  else if (!costing_p)
+	    {
+	      bool ok = vect_transform_slp_perm_load (vinfo, slp_node, dr_chain,
+						      gsi, vf, false, &n_perms,
+						      nullptr, true);
+	      gcc_assert (ok);
+	    }
+	}
       else if (!costing_p)
         {
           if (grouped_load)
@@ -10781,8 +10811,11 @@  vec_num_loop_costing_end:
 
   if (costing_p)
     {
-      if (memory_access_type == VMAT_GATHER_SCATTER
-	  || memory_access_type == VMAT_LOAD_STORE_LANES)
+      gcc_assert (memory_access_type != VMAT_INVARIANT
+		  && memory_access_type != VMAT_ELEMENTWISE
+		  && memory_access_type != VMAT_STRIDED_SLP);
+      if (memory_access_type != VMAT_CONTIGUOUS
+	  && memory_access_type != VMAT_CONTIGUOUS_PERMUTE)
 	{
 	  if (dump_enabled_p ())
 	    dump_printf_loc (MSG_NOTE, vect_location,