[2/3] vect: Move VMAT_LOAD_STORE_LANES handlings from final loop nest

  Hi,

Like commit r14-3214 which moves the handlings on memory
access type VMAT_LOAD_STORE_LANES in vectorizable_load
final loop nest, this one is to deal with the function
vectorizable_store.

Bootstrapped and regtested on x86_64-redhat-linux,
aarch64-linux-gnu and powerpc64{,le}-linux-gnu.

Is it ok for trunk?

BR,
Kewen
-----

gcc/ChangeLog:

	* tree-vect-stmts.cc (vectorizable_store): Move the handlings on
	VMAT_LOAD_STORE_LANES in the final loop nest to its own loop,
	and update the final nest accordingly.
---
 gcc/tree-vect-stmts.cc | 732 ++++++++++++++++++++++-------------------
 1 file changed, 387 insertions(+), 345 deletions(-)

--
2.31.1
  

On Tue, Aug 22, 2023 at 10:49 AM Kewen.Lin <linkw@linux.ibm.com> wrote:
>
> Hi,
>
> Like commit r14-3214 which moves the handlings on memory
> access type VMAT_LOAD_STORE_LANES in vectorizable_load
> final loop nest, this one is to deal with the function
> vectorizable_store.
>
> Bootstrapped and regtested on x86_64-redhat-linux,
> aarch64-linux-gnu and powerpc64{,le}-linux-gnu.
>
> Is it ok for trunk?

OK.

> BR,
> Kewen
> -----
>
> gcc/ChangeLog:
>
>         * tree-vect-stmts.cc (vectorizable_store): Move the handlings on
>         VMAT_LOAD_STORE_LANES in the final loop nest to its own loop,
>         and update the final nest accordingly.
> ---
>  gcc/tree-vect-stmts.cc | 732 ++++++++++++++++++++++-------------------
>  1 file changed, 387 insertions(+), 345 deletions(-)
>
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
> index fcaa4127e52..18f5ebcc09c 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -8779,42 +8779,29 @@ vectorizable_store (vec_info *vinfo,
>    */
>
>    auto_vec<tree> dr_chain (group_size);
> -  auto_vec<tree> result_chain (group_size);
>    auto_vec<tree> vec_masks;
>    tree vec_mask = NULL;
> -  auto_vec<tree> vec_offsets;
>    auto_delete_vec<auto_vec<tree>> gvec_oprnds (group_size);
>    for (i = 0; i < group_size; i++)
>      gvec_oprnds.quick_push (new auto_vec<tree> (ncopies));
> -  auto_vec<tree, 1> vec_oprnds;
> -  for (j = 0; j < ncopies; j++)
> +
> +  if (memory_access_type == VMAT_LOAD_STORE_LANES)
>      {
> -      gimple *new_stmt;
> -      if (j == 0)
> +      gcc_assert (!slp && grouped_store);
> +      for (j = 0; j < ncopies; j++)
>         {
> -          if (slp)
> -            {
> -             /* Get vectorized arguments for SLP_NODE.  */
> -             vect_get_vec_defs (vinfo, stmt_info, slp_node, 1,
> -                                op, &vec_oprnds);
> -              vec_oprnd = vec_oprnds[0];
> -            }
> -          else
> -            {
> -             /* For interleaved stores we collect vectorized defs for all the
> -                stores in the group in DR_CHAIN. DR_CHAIN is then used as an
> -                input to vect_permute_store_chain().
> -
> -                If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
> -                is of size 1.  */
> +         gimple *new_stmt;
> +         if (j == 0)
> +           {
> +             /* For interleaved stores we collect vectorized defs for all
> +                the stores in the group in DR_CHAIN. DR_CHAIN is then used
> +                as an input to vect_permute_store_chain().  */
>               stmt_vec_info next_stmt_info = first_stmt_info;
>               for (i = 0; i < group_size; i++)
>                 {
>                   /* Since gaps are not supported for interleaved stores,
> -                    DR_GROUP_SIZE is the exact number of stmts in the chain.
> -                    Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
> -                    that there is no interleaving, DR_GROUP_SIZE is 1,
> -                    and only one iteration of the loop will be executed.  */
> +                    DR_GROUP_SIZE is the exact number of stmts in the
> +                    chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE.  */
>                   op = vect_get_store_rhs (next_stmt_info);
>                   vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
>                                                  op, gvec_oprnds[i]);
> @@ -8825,66 +8812,37 @@ vectorizable_store (vec_info *vinfo,
>               if (mask)
>                 {
>                   vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
> -                                                mask, &vec_masks, mask_vectype);
> +                                                mask, &vec_masks,
> +                                                mask_vectype);
>                   vec_mask = vec_masks[0];
>                 }
> -           }
>
> -         /* We should have catched mismatched types earlier.  */
> -         gcc_assert (useless_type_conversion_p (vectype,
> -                                                TREE_TYPE (vec_oprnd)));
> -         bool simd_lane_access_p
> -           = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
> -         if (simd_lane_access_p
> -             && !loop_masks
> -             && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
> -             && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
> -             && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
> -             && integer_zerop (DR_INIT (first_dr_info->dr))
> -             && alias_sets_conflict_p (get_alias_set (aggr_type),
> -                                       get_alias_set (TREE_TYPE (ref_type))))
> -           {
> -             dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
> -             dataref_offset = build_int_cst (ref_type, 0);
> +             /* We should have catched mismatched types earlier.  */
> +             gcc_assert (
> +               useless_type_conversion_p (vectype, TREE_TYPE (vec_oprnd)));
> +             dataref_ptr
> +               = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> +                                           NULL, offset, &dummy, gsi,
> +                                           &ptr_incr, false, bump);
>             }
> -         else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -           vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info,
> -                                        slp_node, &gs_info, &dataref_ptr,
> -                                        &vec_offsets);
>           else
> -           dataref_ptr
> -             = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> -                                         simd_lane_access_p ? loop : NULL,
> -                                         offset, &dummy, gsi, &ptr_incr,
> -                                         simd_lane_access_p, bump);
> -       }
> -      else
> -       {
> -         gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> -         /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> -            If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
> -            of size 1.  */
> -         for (i = 0; i < group_size; i++)
>             {
> -             vec_oprnd = (*gvec_oprnds[i])[j];
> -             dr_chain[i] = vec_oprnd;
> +             gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> +             /* DR_CHAIN is then used as an input to
> +                vect_permute_store_chain().  */
> +             for (i = 0; i < group_size; i++)
> +               {
> +                 vec_oprnd = (*gvec_oprnds[i])[j];
> +                 dr_chain[i] = vec_oprnd;
> +               }
> +             if (mask)
> +               vec_mask = vec_masks[j];
> +             dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> +                                            stmt_info, bump);
>             }
> -         if (mask)
> -           vec_mask = vec_masks[j];
> -         if (dataref_offset)
> -           dataref_offset
> -             = int_const_binop (PLUS_EXPR, dataref_offset, bump);
> -         else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> -                                          stmt_info, bump);
> -       }
> -
> -      if (memory_access_type == VMAT_LOAD_STORE_LANES)
> -       {
> -         tree vec_array;
>
>           /* Get an array into which we can store the individual vectors.  */
> -         vec_array = create_vector_array (vectype, vec_num);
> +         tree vec_array = create_vector_array (vectype, vec_num);
>
>           /* Invalidate the current contents of VEC_ARRAY.  This should
>              become an RTL clobber too, which prevents the vector registers
> @@ -8895,8 +8853,8 @@ vectorizable_store (vec_info *vinfo,
>           for (i = 0; i < vec_num; i++)
>             {
>               vec_oprnd = dr_chain[i];
> -             write_vector_array (vinfo, stmt_info,
> -                                 gsi, vec_oprnd, vec_array, i);
> +             write_vector_array (vinfo, stmt_info, gsi, vec_oprnd, vec_array,
> +                                 i);
>             }
>
>           tree final_mask = NULL;
> @@ -8906,8 +8864,8 @@ vectorizable_store (vec_info *vinfo,
>             final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
>                                              ncopies, vectype, j);
>           if (vec_mask)
> -           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> -                                          final_mask, vec_mask, gsi);
> +           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
> +                                          vec_mask, gsi);
>
>           if (lanes_ifn == IFN_MASK_LEN_STORE_LANES)
>             {
> @@ -8955,8 +8913,7 @@ vectorizable_store (vec_info *vinfo,
>               /* Emit:
>                    MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY).  */
>               data_ref = create_array_ref (aggr_type, dataref_ptr, ref_type);
> -             call = gimple_build_call_internal (IFN_STORE_LANES, 1,
> -                                                vec_array);
> +             call = gimple_build_call_internal (IFN_STORE_LANES, 1, vec_array);
>               gimple_call_set_lhs (call, data_ref);
>             }
>           gimple_call_set_nothrow (call, true);
> @@ -8965,301 +8922,386 @@ vectorizable_store (vec_info *vinfo,
>
>           /* Record that VEC_ARRAY is now dead.  */
>           vect_clobber_variable (vinfo, stmt_info, gsi, vec_array);
> +         if (j == 0)
> +           *vec_stmt = new_stmt;
> +         STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt);
>         }
> -      else
> -       {
> -         new_stmt = NULL;
> -         if (grouped_store)
> -           /* Permute.  */
> -           vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info,
> -                                     gsi, &result_chain);
>
> -         stmt_vec_info next_stmt_info = first_stmt_info;
> -         for (i = 0; i < vec_num; i++)
> -           {
> -             unsigned misalign;
> -             unsigned HOST_WIDE_INT align;
> +      return true;
> +    }
>
> -             tree final_mask = NULL_TREE;
> -             tree final_len = NULL_TREE;
> -             tree bias = NULL_TREE;
> -             if (loop_masks)
> -               final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> -                                                vec_num * ncopies,
> -                                                vectype, vec_num * j + i);
> -             if (vec_mask)
> -               final_mask = prepare_vec_mask (loop_vinfo, mask_vectype,
> -                                              final_mask, vec_mask, gsi);
> +  auto_vec<tree> result_chain (group_size);
> +  auto_vec<tree> vec_offsets;
> +  auto_vec<tree, 1> vec_oprnds;
> +  for (j = 0; j < ncopies; j++)
> +    {
> +      gimple *new_stmt;
> +      if (j == 0)
> +       {
> +         if (slp)
> +           {
> +             /* Get vectorized arguments for SLP_NODE.  */
> +             vect_get_vec_defs (vinfo, stmt_info, slp_node, 1, op,
> +                                &vec_oprnds);
> +             vec_oprnd = vec_oprnds[0];
> +           }
> +         else
> +           {
> +             /* For interleaved stores we collect vectorized defs for all the
> +                stores in the group in DR_CHAIN. DR_CHAIN is then used as an
> +                input to vect_permute_store_chain().
>
> -             if (memory_access_type == VMAT_GATHER_SCATTER
> -                 && gs_info.ifn != IFN_LAST)
> +                If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
> +                is of size 1.  */
> +             stmt_vec_info next_stmt_info = first_stmt_info;
> +             for (i = 0; i < group_size; i++)
>                 {
> -                 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> -                   vec_offset = vec_offsets[vec_num * j + i];
> -                 tree scale = size_int (gs_info.scale);
> -
> -                 if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
> -                   {
> -                     if (loop_lens)
> -                       final_len
> -                         = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> -                                              vec_num * ncopies, vectype,
> -                                              vec_num * j + i, 1);
> -                     else
> -                       final_len
> -                         = build_int_cst (sizetype,
> -                                          TYPE_VECTOR_SUBPARTS (vectype));
> -                     signed char biasval
> -                       = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> -                     bias = build_int_cst (intQI_type_node, biasval);
> -                     if (!final_mask)
> -                       {
> -                         mask_vectype = truth_type_for (vectype);
> -                         final_mask = build_minus_one_cst (mask_vectype);
> -                       }
> -                   }
> -
> -                 gcall *call;
> -                 if (final_len && final_mask)
> -                   call
> -                     = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
> -                                                   7, dataref_ptr, vec_offset,
> -                                                   scale, vec_oprnd, final_mask,
> -                                                   final_len, bias);
> -                 else if (final_mask)
> -                   call = gimple_build_call_internal
> -                     (IFN_MASK_SCATTER_STORE, 5, dataref_ptr, vec_offset,
> -                      scale, vec_oprnd, final_mask);
> -                 else
> -                   call = gimple_build_call_internal
> -                     (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset,
> -                      scale, vec_oprnd);
> -                 gimple_call_set_nothrow (call, true);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> -                 new_stmt = call;
> -                 break;
> +                 /* Since gaps are not supported for interleaved stores,
> +                    DR_GROUP_SIZE is the exact number of stmts in the chain.
> +                    Therefore, NEXT_STMT_INFO can't be NULL_TREE.  In case
> +                    that there is no interleaving, DR_GROUP_SIZE is 1,
> +                    and only one iteration of the loop will be executed.  */
> +                 op = vect_get_store_rhs (next_stmt_info);
> +                 vect_get_vec_defs_for_operand (vinfo, next_stmt_info, ncopies,
> +                                                op, gvec_oprnds[i]);
> +                 vec_oprnd = (*gvec_oprnds[i])[0];
> +                 dr_chain.quick_push (vec_oprnd);
> +                 next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
>                 }
> -             else if (memory_access_type == VMAT_GATHER_SCATTER)
> +             if (mask)
>                 {
> -                 /* Emulated scatter.  */
> -                 gcc_assert (!final_mask);
> -                 unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
> -                 unsigned HOST_WIDE_INT const_offset_nunits
> -                   = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype)
> -                       .to_constant ();
> -                 vec<constructor_elt, va_gc> *ctor_elts;
> -                 vec_alloc (ctor_elts, const_nunits);
> -                 gimple_seq stmts = NULL;
> -                 tree elt_type = TREE_TYPE (vectype);
> -                 unsigned HOST_WIDE_INT elt_size
> -                   = tree_to_uhwi (TYPE_SIZE (elt_type));
> -                 /* We support offset vectors with more elements
> -                    than the data vector for now.  */
> -                 unsigned HOST_WIDE_INT factor
> -                   = const_offset_nunits / const_nunits;
> -                 vec_offset = vec_offsets[j / factor];
> -                 unsigned elt_offset = (j % factor) * const_nunits;
> -                 tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
> -                 tree scale = size_int (gs_info.scale);
> -                 align = get_object_alignment (DR_REF (first_dr_info->dr));
> -                 tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
> -                 for (unsigned k = 0; k < const_nunits; ++k)
> -                   {
> -                     /* Compute the offsetted pointer.  */
> -                     tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
> -                                             bitsize_int (k + elt_offset));
> -                     tree idx = gimple_build (&stmts, BIT_FIELD_REF,
> -                                              idx_type, vec_offset,
> -                                              TYPE_SIZE (idx_type), boff);
> -                     idx = gimple_convert (&stmts, sizetype, idx);
> -                     idx = gimple_build (&stmts, MULT_EXPR,
> -                                         sizetype, idx, scale);
> -                     tree ptr = gimple_build (&stmts, PLUS_EXPR,
> -                                              TREE_TYPE (dataref_ptr),
> -                                              dataref_ptr, idx);
> -                     ptr = gimple_convert (&stmts, ptr_type_node, ptr);
> -                     /* Extract the element to be stored.  */
> -                     tree elt = gimple_build (&stmts, BIT_FIELD_REF,
> -                                              TREE_TYPE (vectype), vec_oprnd,
> -                                              TYPE_SIZE (elt_type),
> -                                              bitsize_int (k * elt_size));
> -                     gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
> -                     stmts = NULL;
> -                     tree ref = build2 (MEM_REF, ltype, ptr,
> -                                        build_int_cst (ref_type, 0));
> -                     new_stmt = gimple_build_assign (ref, elt);
> -                     vect_finish_stmt_generation (vinfo, stmt_info,
> -                                                  new_stmt, gsi);
> -                   }
> -                 break;
> +                 vect_get_vec_defs_for_operand (vinfo, stmt_info, ncopies,
> +                                                mask, &vec_masks,
> +                                                mask_vectype);
> +                 vec_mask = vec_masks[0];
>                 }
> +           }
>
> -             if (i > 0)
> -               /* Bump the vector pointer.  */
> -               dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr,
> -                                              gsi, stmt_info, bump);
> +         /* We should have catched mismatched types earlier.  */
> +         gcc_assert (useless_type_conversion_p (vectype,
> +                                                TREE_TYPE (vec_oprnd)));
> +         bool simd_lane_access_p
> +           = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info) != 0;
> +         if (simd_lane_access_p
> +             && !loop_masks
> +             && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
> +             && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
> +             && integer_zerop (get_dr_vinfo_offset (vinfo, first_dr_info))
> +             && integer_zerop (DR_INIT (first_dr_info->dr))
> +             && alias_sets_conflict_p (get_alias_set (aggr_type),
> +                                       get_alias_set (TREE_TYPE (ref_type))))
> +           {
> +             dataref_ptr = unshare_expr (DR_BASE_ADDRESS (first_dr_info->dr));
> +             dataref_offset = build_int_cst (ref_type, 0);
> +           }
> +         else if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +           vect_get_gather_scatter_ops (loop_vinfo, loop, stmt_info, slp_node,
> +                                        &gs_info, &dataref_ptr, &vec_offsets);
> +         else
> +           dataref_ptr
> +             = vect_create_data_ref_ptr (vinfo, first_stmt_info, aggr_type,
> +                                         simd_lane_access_p ? loop : NULL,
> +                                         offset, &dummy, gsi, &ptr_incr,
> +                                         simd_lane_access_p, bump);
> +       }
> +      else
> +       {
> +         gcc_assert (!LOOP_VINFO_USING_SELECT_VL_P (loop_vinfo));
> +         /* DR_CHAIN is then used as an input to vect_permute_store_chain().
> +            If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
> +            of size 1.  */
> +         for (i = 0; i < group_size; i++)
> +           {
> +             vec_oprnd = (*gvec_oprnds[i])[j];
> +             dr_chain[i] = vec_oprnd;
> +           }
> +         if (mask)
> +           vec_mask = vec_masks[j];
> +         if (dataref_offset)
> +           dataref_offset = int_const_binop (PLUS_EXPR, dataref_offset, bump);
> +         else if (!STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> +                                          stmt_info, bump);
> +       }
>
> -             if (slp)
> -               vec_oprnd = vec_oprnds[i];
> -             else if (grouped_store)
> -               /* For grouped stores vectorized defs are interleaved in
> -                  vect_permute_store_chain().  */
> -               vec_oprnd = result_chain[i];
> +      new_stmt = NULL;
> +      if (grouped_store)
> +       /* Permute.  */
> +       vect_permute_store_chain (vinfo, dr_chain, group_size, stmt_info, gsi,
> +                                 &result_chain);
>
> -             align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
> -             if (alignment_support_scheme == dr_aligned)
> -               misalign = 0;
> -             else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> -               {
> -                 align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
> -                 misalign = 0;
> -               }
> -             else
> -               misalign = misalignment;
> -             if (dataref_offset == NULL_TREE
> -                 && TREE_CODE (dataref_ptr) == SSA_NAME)
> -               set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> -                                       misalign);
> -             align = least_bit_hwi (misalign | align);
> -
> -             if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
> -               {
> -                 tree perm_mask = perm_mask_for_reverse (vectype);
> -                 tree perm_dest = vect_create_destination_var
> -                   (vect_get_store_rhs (stmt_info), vectype);
> -                 tree new_temp = make_ssa_name (perm_dest);
> -
> -                 /* Generate the permute statement.  */
> -                 gimple *perm_stmt
> -                   = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
> -                                          vec_oprnd, perm_mask);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
> -
> -                 perm_stmt = SSA_NAME_DEF_STMT (new_temp);
> -                 vec_oprnd = new_temp;
> -               }
> +      stmt_vec_info next_stmt_info = first_stmt_info;
> +      for (i = 0; i < vec_num; i++)
> +       {
> +         unsigned misalign;
> +         unsigned HOST_WIDE_INT align;
>
> -             /* Compute IFN when LOOP_LENS or final_mask valid.  */
> -             machine_mode vmode = TYPE_MODE (vectype);
> -             machine_mode new_vmode = vmode;
> -             internal_fn partial_ifn = IFN_LAST;
> -             if (loop_lens)
> -               {
> -                 opt_machine_mode new_ovmode
> -                   = get_len_load_store_mode (vmode, false, &partial_ifn);
> -                 new_vmode = new_ovmode.require ();
> -                 unsigned factor
> -                   = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
> -                 final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> -                                                vec_num * ncopies, vectype,
> -                                                vec_num * j + i, factor);
> -               }
> -             else if (final_mask)
> -               {
> -                 if (!can_vec_mask_load_store_p (vmode,
> -                                                 TYPE_MODE (TREE_TYPE (final_mask)),
> -                                                 false, &partial_ifn))
> -                   gcc_unreachable ();
> -               }
> +         tree final_mask = NULL_TREE;
> +         tree final_len = NULL_TREE;
> +         tree bias = NULL_TREE;
> +         if (loop_masks)
> +           final_mask = vect_get_loop_mask (loop_vinfo, gsi, loop_masks,
> +                                            vec_num * ncopies, vectype,
> +                                            vec_num * j + i);
> +         if (vec_mask)
> +           final_mask = prepare_vec_mask (loop_vinfo, mask_vectype, final_mask,
> +                                          vec_mask, gsi);
>
> -             if (partial_ifn == IFN_MASK_LEN_STORE)
> +         if (memory_access_type == VMAT_GATHER_SCATTER
> +             && gs_info.ifn != IFN_LAST)
> +           {
> +             if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
> +               vec_offset = vec_offsets[vec_num * j + i];
> +             tree scale = size_int (gs_info.scale);
> +
> +             if (gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE)
>                 {
> -                 if (!final_len)
> -                   {
> -                     /* Pass VF value to 'len' argument of
> -                        MASK_LEN_STORE if LOOP_LENS is invalid.  */
> -                     final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
> -                   }
> +                 if (loop_lens)
> +                   final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> +                                                  vec_num * ncopies, vectype,
> +                                                  vec_num * j + i, 1);
> +                 else
> +                   final_len = build_int_cst (sizetype,
> +                                              TYPE_VECTOR_SUBPARTS (vectype));
> +                 signed char biasval
> +                   = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> +                 bias = build_int_cst (intQI_type_node, biasval);
>                   if (!final_mask)
>                     {
> -                     /* Pass all ones value to 'mask' argument of
> -                        MASK_LEN_STORE if final_mask is invalid.  */
>                       mask_vectype = truth_type_for (vectype);
>                       final_mask = build_minus_one_cst (mask_vectype);
>                     }
>                 }
> -             if (final_len)
> -               {
> -                 signed char biasval
> -                   = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
>
> -                 bias = build_int_cst (intQI_type_node, biasval);
> +             gcall *call;
> +             if (final_len && final_mask)
> +               call = gimple_build_call_internal (IFN_MASK_LEN_SCATTER_STORE,
> +                                                  7, dataref_ptr, vec_offset,
> +                                                  scale, vec_oprnd, final_mask,
> +                                                  final_len, bias);
> +             else if (final_mask)
> +               call
> +                 = gimple_build_call_internal (IFN_MASK_SCATTER_STORE, 5,
> +                                               dataref_ptr, vec_offset, scale,
> +                                               vec_oprnd, final_mask);
> +             else
> +               call = gimple_build_call_internal (IFN_SCATTER_STORE, 4,
> +                                                  dataref_ptr, vec_offset,
> +                                                  scale, vec_oprnd);
> +             gimple_call_set_nothrow (call, true);
> +             vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> +             new_stmt = call;
> +             break;
> +           }
> +         else if (memory_access_type == VMAT_GATHER_SCATTER)
> +           {
> +             /* Emulated scatter.  */
> +             gcc_assert (!final_mask);
> +             unsigned HOST_WIDE_INT const_nunits = nunits.to_constant ();
> +             unsigned HOST_WIDE_INT const_offset_nunits
> +               = TYPE_VECTOR_SUBPARTS (gs_info.offset_vectype).to_constant ();
> +             vec<constructor_elt, va_gc> *ctor_elts;
> +             vec_alloc (ctor_elts, const_nunits);
> +             gimple_seq stmts = NULL;
> +             tree elt_type = TREE_TYPE (vectype);
> +             unsigned HOST_WIDE_INT elt_size
> +               = tree_to_uhwi (TYPE_SIZE (elt_type));
> +             /* We support offset vectors with more elements
> +                than the data vector for now.  */
> +             unsigned HOST_WIDE_INT factor
> +               = const_offset_nunits / const_nunits;
> +             vec_offset = vec_offsets[j / factor];
> +             unsigned elt_offset = (j % factor) * const_nunits;
> +             tree idx_type = TREE_TYPE (TREE_TYPE (vec_offset));
> +             tree scale = size_int (gs_info.scale);
> +             align = get_object_alignment (DR_REF (first_dr_info->dr));
> +             tree ltype = build_aligned_type (TREE_TYPE (vectype), align);
> +             for (unsigned k = 0; k < const_nunits; ++k)
> +               {
> +                 /* Compute the offsetted pointer.  */
> +                 tree boff = size_binop (MULT_EXPR, TYPE_SIZE (idx_type),
> +                                         bitsize_int (k + elt_offset));
> +                 tree idx
> +                   = gimple_build (&stmts, BIT_FIELD_REF, idx_type, vec_offset,
> +                                   TYPE_SIZE (idx_type), boff);
> +                 idx = gimple_convert (&stmts, sizetype, idx);
> +                 idx = gimple_build (&stmts, MULT_EXPR, sizetype, idx, scale);
> +                 tree ptr
> +                   = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (dataref_ptr),
> +                                   dataref_ptr, idx);
> +                 ptr = gimple_convert (&stmts, ptr_type_node, ptr);
> +                 /* Extract the element to be stored.  */
> +                 tree elt
> +                   = gimple_build (&stmts, BIT_FIELD_REF, TREE_TYPE (vectype),
> +                                   vec_oprnd, TYPE_SIZE (elt_type),
> +                                   bitsize_int (k * elt_size));
> +                 gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
> +                 stmts = NULL;
> +                 tree ref
> +                   = build2 (MEM_REF, ltype, ptr, build_int_cst (ref_type, 0));
> +                 new_stmt = gimple_build_assign (ref, elt);
> +                 vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
>                 }
> +             break;
> +           }
>
> -             /* Arguments are ready.  Create the new vector stmt.  */
> -             if (final_len)
> -               {
> -                 gcall *call;
> -                 tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> -                 /* Need conversion if it's wrapped with VnQI.  */
> -                 if (vmode != new_vmode)
> -                   {
> -                     tree new_vtype
> -                       = build_vector_type_for_mode (unsigned_intQI_type_node,
> -                                                     new_vmode);
> -                     tree var
> -                       = vect_get_new_ssa_name (new_vtype, vect_simple_var);
> -                     vec_oprnd
> -                       = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
> -                     gassign *new_stmt
> -                       = gimple_build_assign (var, VIEW_CONVERT_EXPR,
> -                                              vec_oprnd);
> -                     vect_finish_stmt_generation (vinfo, stmt_info, new_stmt,
> -                                                  gsi);
> -                     vec_oprnd = var;
> -                   }
> +         if (i > 0)
> +           /* Bump the vector pointer.  */
> +           dataref_ptr = bump_vector_ptr (vinfo, dataref_ptr, ptr_incr, gsi,
> +                                          stmt_info, bump);
>
> -                 if (partial_ifn == IFN_MASK_LEN_STORE)
> -                   call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
> -                                                      dataref_ptr, ptr,
> -                                                      final_mask, final_len,
> -                                                      bias, vec_oprnd);
> -                 else
> -                   call
> -                     = gimple_build_call_internal (IFN_LEN_STORE, 5,
> -                                                   dataref_ptr, ptr,
> -                                                   final_len, bias,
> -                                                   vec_oprnd);
> -                 gimple_call_set_nothrow (call, true);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> -                 new_stmt = call;
> +         if (slp)
> +           vec_oprnd = vec_oprnds[i];
> +         else if (grouped_store)
> +           /* For grouped stores vectorized defs are interleaved in
> +              vect_permute_store_chain().  */
> +           vec_oprnd = result_chain[i];
> +
> +         align = known_alignment (DR_TARGET_ALIGNMENT (first_dr_info));
> +         if (alignment_support_scheme == dr_aligned)
> +           misalign = 0;
> +         else if (misalignment == DR_MISALIGNMENT_UNKNOWN)
> +           {
> +             align = dr_alignment (vect_dr_behavior (vinfo, first_dr_info));
> +             misalign = 0;
> +           }
> +         else
> +           misalign = misalignment;
> +         if (dataref_offset == NULL_TREE
> +             && TREE_CODE (dataref_ptr) == SSA_NAME)
> +           set_ptr_info_alignment (get_ptr_info (dataref_ptr), align,
> +                                   misalign);
> +         align = least_bit_hwi (misalign | align);
> +
> +         if (memory_access_type == VMAT_CONTIGUOUS_REVERSE)
> +           {
> +             tree perm_mask = perm_mask_for_reverse (vectype);
> +             tree perm_dest
> +               = vect_create_destination_var (vect_get_store_rhs (stmt_info),
> +                                              vectype);
> +             tree new_temp = make_ssa_name (perm_dest);
> +
> +             /* Generate the permute statement.  */
> +             gimple *perm_stmt
> +               = gimple_build_assign (new_temp, VEC_PERM_EXPR, vec_oprnd,
> +                                      vec_oprnd, perm_mask);
> +             vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
> +
> +             perm_stmt = SSA_NAME_DEF_STMT (new_temp);
> +             vec_oprnd = new_temp;
> +           }
> +
> +         /* Compute IFN when LOOP_LENS or final_mask valid.  */
> +         machine_mode vmode = TYPE_MODE (vectype);
> +         machine_mode new_vmode = vmode;
> +         internal_fn partial_ifn = IFN_LAST;
> +         if (loop_lens)
> +           {
> +             opt_machine_mode new_ovmode
> +               = get_len_load_store_mode (vmode, false, &partial_ifn);
> +             new_vmode = new_ovmode.require ();
> +             unsigned factor
> +               = (new_ovmode == vmode) ? 1 : GET_MODE_UNIT_SIZE (vmode);
> +             final_len = vect_get_loop_len (loop_vinfo, gsi, loop_lens,
> +                                            vec_num * ncopies, vectype,
> +                                            vec_num * j + i, factor);
> +           }
> +         else if (final_mask)
> +           {
> +             if (!can_vec_mask_load_store_p (
> +                   vmode, TYPE_MODE (TREE_TYPE (final_mask)), false,
> +                   &partial_ifn))
> +               gcc_unreachable ();
> +           }
> +
> +         if (partial_ifn == IFN_MASK_LEN_STORE)
> +           {
> +             if (!final_len)
> +               {
> +                 /* Pass VF value to 'len' argument of
> +                    MASK_LEN_STORE if LOOP_LENS is invalid.  */
> +                 final_len = size_int (TYPE_VECTOR_SUBPARTS (vectype));
>                 }
> -             else if (final_mask)
> +             if (!final_mask)
>                 {
> -                 tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> -                 gcall *call
> -                   = gimple_build_call_internal (IFN_MASK_STORE, 4,
> -                                                 dataref_ptr, ptr,
> -                                                 final_mask, vec_oprnd);
> -                 gimple_call_set_nothrow (call, true);
> -                 vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> -                 new_stmt = call;
> +                 /* Pass all ones value to 'mask' argument of
> +                    MASK_LEN_STORE if final_mask is invalid.  */
> +                 mask_vectype = truth_type_for (vectype);
> +                 final_mask = build_minus_one_cst (mask_vectype);
>                 }
> -             else
> +           }
> +         if (final_len)
> +           {
> +             signed char biasval
> +               = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> +
> +             bias = build_int_cst (intQI_type_node, biasval);
> +           }
> +
> +         /* Arguments are ready.  Create the new vector stmt.  */
> +         if (final_len)
> +           {
> +             gcall *call;
> +             tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> +             /* Need conversion if it's wrapped with VnQI.  */
> +             if (vmode != new_vmode)
>                 {
> -                 data_ref = fold_build2 (MEM_REF, vectype,
> -                                         dataref_ptr,
> -                                         dataref_offset
> -                                         ? dataref_offset
> -                                         : build_int_cst (ref_type, 0));
> -                 if (alignment_support_scheme == dr_aligned)
> -                   ;
> -                 else
> -                   TREE_TYPE (data_ref)
> -                     = build_aligned_type (TREE_TYPE (data_ref),
> -                                           align * BITS_PER_UNIT);
> -                 vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
> -                 new_stmt = gimple_build_assign (data_ref, vec_oprnd);
> +                 tree new_vtype
> +                   = build_vector_type_for_mode (unsigned_intQI_type_node,
> +                                                 new_vmode);
> +                 tree var = vect_get_new_ssa_name (new_vtype, vect_simple_var);
> +                 vec_oprnd = build1 (VIEW_CONVERT_EXPR, new_vtype, vec_oprnd);
> +                 gassign *new_stmt
> +                   = gimple_build_assign (var, VIEW_CONVERT_EXPR, vec_oprnd);
>                   vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> +                 vec_oprnd = var;
>                 }
>
> -             if (slp)
> -               continue;
> -
> -             next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> -             if (!next_stmt_info)
> -               break;
> +             if (partial_ifn == IFN_MASK_LEN_STORE)
> +               call = gimple_build_call_internal (IFN_MASK_LEN_STORE, 6,
> +                                                  dataref_ptr, ptr, final_mask,
> +                                                  final_len, bias, vec_oprnd);
> +             else
> +               call = gimple_build_call_internal (IFN_LEN_STORE, 5,
> +                                                  dataref_ptr, ptr, final_len,
> +                                                  bias, vec_oprnd);
> +             gimple_call_set_nothrow (call, true);
> +             vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> +             new_stmt = call;
> +           }
> +         else if (final_mask)
> +           {
> +             tree ptr = build_int_cst (ref_type, align * BITS_PER_UNIT);
> +             gcall *call
> +               = gimple_build_call_internal (IFN_MASK_STORE, 4, dataref_ptr,
> +                                             ptr, final_mask, vec_oprnd);
> +             gimple_call_set_nothrow (call, true);
> +             vect_finish_stmt_generation (vinfo, stmt_info, call, gsi);
> +             new_stmt = call;
> +           }
> +         else
> +           {
> +             data_ref
> +               = fold_build2 (MEM_REF, vectype, dataref_ptr,
> +                              dataref_offset ? dataref_offset
> +                                             : build_int_cst (ref_type, 0));
> +             if (alignment_support_scheme == dr_aligned)
> +               ;
> +             else
> +               TREE_TYPE (data_ref)
> +                 = build_aligned_type (TREE_TYPE (data_ref),
> +                                       align * BITS_PER_UNIT);
> +             vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr));
> +             new_stmt = gimple_build_assign (data_ref, vec_oprnd);
> +             vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
>             }
> +
> +         if (slp)
> +           continue;
> +
> +         next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
> +         if (!next_stmt_info)
> +           break;
>         }
>        if (!slp)
>         {
> --
> 2.31.1