[02/10] vect: Move vect_model_store_cost next to the transform in vectorizable_store

  This patch is an initial patch to move costing next to the
transform, it still adopts vect_model_store_cost for costing
but moves and duplicates it down according to the handlings
of different vect_memory_access_types or some special
handling need, hope it can make the subsequent patches easy
to review.  This patch should not have any functional
changes.

gcc/ChangeLog:

	* tree-vect-stmts.cc (vectorizable_store): Move and duplicate the call
	to vect_model_store_cost down to some different transform paths
	according to the handlings of different vect_memory_access_types
	or some special handling need.
---
 gcc/tree-vect-stmts.cc | 79 ++++++++++++++++++++++++++++++++----------
 1 file changed, 60 insertions(+), 19 deletions(-)
  

On Thu, Sep 14, 2023 at 5:12 AM Kewen Lin <linkw@linux.ibm.com> wrote:
>
> This patch is an initial patch to move costing next to the
> transform, it still adopts vect_model_store_cost for costing
> but moves and duplicates it down according to the handlings
> of different vect_memory_access_types or some special
> handling need, hope it can make the subsequent patches easy
> to review.  This patch should not have any functional
> changes.

OK

> gcc/ChangeLog:
>
>         * tree-vect-stmts.cc (vectorizable_store): Move and duplicate the call
>         to vect_model_store_cost down to some different transform paths
>         according to the handlings of different vect_memory_access_types
>         or some special handling need.
> ---
>  gcc/tree-vect-stmts.cc | 79 ++++++++++++++++++++++++++++++++----------
>  1 file changed, 60 insertions(+), 19 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index a5caaf0bca2..36f7c5b9f4b 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -8372,7 +8372,8 @@ vectorizable_store (vec_info *vinfo,
>         return false;
>      }
>
> -  if (!vec_stmt) /* transformation not required.  */
> +  bool costing_p = !vec_stmt;
> +  if (costing_p) /* transformation not required.  */
>      {
>        STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) = memory_access_type;
>
> @@ -8401,11 +8402,6 @@ vectorizable_store (vec_info *vinfo,
>                          "Vectorizing an unaligned access.\n");
>
>        STMT_VINFO_TYPE (stmt_info) = store_vec_info_type;
> -      vect_model_store_cost (vinfo, stmt_info, ncopies,
> -                            memory_access_type, &gs_info,
> -                            alignment_support_scheme,
> -                            misalignment, vls_type, slp_node, cost_vec);
> -      return true;
>      }
>    gcc_assert (memory_access_type == STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info));
>
> @@ -8415,12 +8411,27 @@ vectorizable_store (vec_info *vinfo,
>
>    if (memory_access_type == VMAT_GATHER_SCATTER && gs_info.decl)
>      {
> -      vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
> -                                     &gs_info, mask);
> +      if (costing_p)
> +       vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                              &gs_info, alignment_support_scheme, misalignment,
> +                              vls_type, slp_node, cost_vec);
> +      else
> +       vect_build_scatter_store_calls (vinfo, stmt_info, gsi, vec_stmt,
> +                                       &gs_info, mask);
>        return true;
>      }
>    else if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) >= 3)
> -    return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
> +    {
> +      gcc_assert (memory_access_type == VMAT_CONTIGUOUS);
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
> +      return vectorizable_scan_store (vinfo, stmt_info, gsi, vec_stmt, ncopies);
> +    }
>
>    if (grouped_store)
>      {
> @@ -8449,13 +8460,21 @@ vectorizable_store (vec_info *vinfo,
>    else
>      ref_type = reference_alias_ptr_type (DR_REF (first_dr_info->dr));
>
> -  if (dump_enabled_p ())
> -    dump_printf_loc (MSG_NOTE, vect_location,
> -                     "transform store. ncopies = %d\n", ncopies);
> +  if (!costing_p && dump_enabled_p ())
> +    dump_printf_loc (MSG_NOTE, vect_location, "transform store. ncopies = %d\n",
> +                    ncopies);
>
>    if (memory_access_type == VMAT_ELEMENTWISE
>        || memory_access_type == VMAT_STRIDED_SLP)
>      {
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
> +
>        gimple_stmt_iterator incr_gsi;
>        bool insert_after;
>        gimple *incr;
> @@ -8718,8 +8737,9 @@ vectorizable_store (vec_info *vinfo,
>    else if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        aggr_type = elem_type;
> -      vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
> -                                      &bump, &vec_offset, loop_lens);
> +      if (!costing_p)
> +       vect_get_strided_load_store_ops (stmt_info, loop_vinfo, gsi, &gs_info,
> +                                        &bump, &vec_offset, loop_lens);
>      }
>    else
>      {
> @@ -8731,7 +8751,7 @@ vectorizable_store (vec_info *vinfo,
>                                           memory_access_type, loop_lens);
>      }
>
> -  if (mask)
> +  if (mask && !costing_p)
>      LOOP_VINFO_HAS_MASK_STORE (loop_vinfo) = true;
>
>    /* In case the vectorization factor (VF) is bigger than the number
> @@ -8782,6 +8802,13 @@ vectorizable_store (vec_info *vinfo,
>    if (memory_access_type == VMAT_LOAD_STORE_LANES)
>      {
>        gcc_assert (!slp && grouped_store);
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
>        for (j = 0; j < ncopies; j++)
>         {
>           gimple *new_stmt;
> @@ -8927,6 +8954,13 @@ vectorizable_store (vec_info *vinfo,
>    if (memory_access_type == VMAT_GATHER_SCATTER)
>      {
>        gcc_assert (!slp && !grouped_store);
> +      if (costing_p)
> +       {
> +         vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                                &gs_info, alignment_support_scheme,
> +                                misalignment, vls_type, slp_node, cost_vec);
> +         return true;
> +       }
>        auto_vec<tree> vec_offsets;
>        for (j = 0; j < ncopies; j++)
>         {
> @@ -9091,7 +9125,7 @@ vectorizable_store (vec_info *vinfo,
>    for (j = 0; j < ncopies; j++)
>      {
>        gimple *new_stmt;
> -      if (j == 0)
> +      if (j == 0 && !costing_p)
>         {
>           if (slp)
>             {
> @@ -9158,7 +9192,7 @@ vectorizable_store (vec_info *vinfo,
>                                           offset, &dummy, gsi, &ptr_incr,
>                                           simd_lane_access_p, bump);
>         }
> -      else
> +      else if (!costing_p)
>         {
>           gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
>           /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> @@ -9179,7 +9213,7 @@ vectorizable_store (vec_info *vinfo,
>         }
>
>        new_stmt = NULL;
> -      if (grouped_store)
> +      if (!costing_p && grouped_store)
>         /* Permute.  */
>         vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
>                                   &result_chain);
> @@ -9187,6 +9221,8 @@ vectorizable_store (vec_info *vinfo,
>        stmt_vec_info next_stmt_info = first_stmt_info;
>        for (i = 0; i < vec_num; i++)
>         {
> +         if (costing_p)
> +           continue;
>           unsigned misalign;
>           unsigned HOST_WIDE_INT align;
>
> @@ -9361,7 +9397,7 @@ vectorizable_store (vec_info *vinfo,
>           if (!next_stmt_info)
>             break;
>         }
> -      if (!slp)
> +      if (!slp && !costing_p)
>         {
>           if (j == 0)
>             *vec_stmt = new_stmt;
> @@ -9369,6 +9405,11 @@ vectorizable_store (vec_info *vinfo,
>         }
>      }
>
> +  if (costing_p)
> +    vect_model_store_cost (vinfo, stmt_info, ncopies, memory_access_type,
> +                          &gs_info, alignment_support_scheme, misalignment,
> +                          vls_type, slp_node, cost_vec);
> +
>    return true;
>  }
>
> --
> 2.31.1
>