[vect] Don't peel nonlinear iv(mult or shift) for epilog when vf is not constant.

  Normally when vf is not constant, it will be prevented by
vectorizable_nonlinear_inductions, but for this case, it failed going
into

    if (STMT_VINFO_RELEVANT_P (stmt_info))
      {
	need_to_vectorize = true;
	if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
	   && ! PURE_SLP_STMT (stmt_info))
	  ok = vectorizable_induction (loop_vinfo,
				       stmt_info, NULL, NULL,
				       &cost_vec);

since the iv is never used outside of the loop, and will be dce later, so
vectorizer doesn't bother checking if it's vectorizable. it's
true but hit gcc_assert in vect_can_peel_nonlinear_iv_p when vf is not
constant. One solution is ignoring the nonlinear iv peeling if it's
!STMT_VINFO_RELEVANT_P (stmt_info) just like the upper code, the other
solution is returning false earlier in the
vect_can_peel_nonlinear_iv_p when vf is not constant, the patch chooses
the second incase there's other cases using vect_can_advance_ivs_p which
calls vect_can_peel_nonlinear_iv_p.

Also remove vect_can_peel_nonlinear_iv_p from vectorizable_nonlinear_inductions.

Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,} and aarch64-linux-gnu{-m32,}
Ok for trunk?

gcc/ChangeLog:

	PR tree-optimization/108601
	* tree-vectorizer.h (vect_can_peel_nonlinear_iv_p): Remove declare.
	* tree-vect-loop.cc
	(vectorizable_nonlinear_induction): Remove
	vect_can_peel_nonlinear_iv_p.
	(vect_can_peel_nonlinear_iv_p): Don't peel
	nonlinear iv(mult or shift) for epilog when vf is not
	constant and moved the defination to ..
	* tree-vect-loop-manip.cc (vect_can_peel_nonlinear_iv_p):
	.. Here.

gcc/testsuite/ChangeLog:

	* gcc.target/aarch64/pr108601.c: New test.
---
 gcc/testsuite/gcc.target/aarch64/pr108601.c | 11 +++++
 gcc/tree-vect-loop-manip.cc                 | 44 ++++++++++++++++++++
 gcc/tree-vect-loop.cc                       | 46 ---------------------
 gcc/tree-vectorizer.h                       |  3 --
 4 files changed, 55 insertions(+), 49 deletions(-)
 create mode 100644 gcc/testsuite/gcc.target/aarch64/pr108601.c
  

On Thu, 2 Feb 2023, liuhongt wrote:

> Normally when vf is not constant, it will be prevented by
> vectorizable_nonlinear_inductions, but for this case, it failed going
> into
> 
>     if (STMT_VINFO_RELEVANT_P (stmt_info))
>       {
> 	need_to_vectorize = true;
> 	if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def
> 	   && ! PURE_SLP_STMT (stmt_info))
> 	  ok = vectorizable_induction (loop_vinfo,
> 				       stmt_info, NULL, NULL,
> 				       &cost_vec);
> 
> since the iv is never used outside of the loop, and will be dce later, so
> vectorizer doesn't bother checking if it's vectorizable. it's
> true but hit gcc_assert in vect_can_peel_nonlinear_iv_p when vf is not
> constant. One solution is ignoring the nonlinear iv peeling if it's
> !STMT_VINFO_RELEVANT_P (stmt_info) just like the upper code, the other
> solution is returning false earlier in the
> vect_can_peel_nonlinear_iv_p when vf is not constant, the patch chooses
> the second incase there's other cases using vect_can_advance_ivs_p which
> calls vect_can_peel_nonlinear_iv_p.
> 
> Also remove vect_can_peel_nonlinear_iv_p from vectorizable_nonlinear_inductions.
> 
> Bootstrapped and regtested on x86_64-pc-linux-gnu{-m32,} and aarch64-linux-gnu{-m32,}
> Ok for trunk?

OK.

Thanks,
Richard.

> gcc/ChangeLog:
> 
> 	PR tree-optimization/108601
> 	* tree-vectorizer.h (vect_can_peel_nonlinear_iv_p): Remove declare.
> 	* tree-vect-loop.cc
> 	(vectorizable_nonlinear_induction): Remove
> 	vect_can_peel_nonlinear_iv_p.
> 	(vect_can_peel_nonlinear_iv_p): Don't peel
> 	nonlinear iv(mult or shift) for epilog when vf is not
> 	constant and moved the defination to ..
> 	* tree-vect-loop-manip.cc (vect_can_peel_nonlinear_iv_p):
> 	.. Here.
> 
> gcc/testsuite/ChangeLog:
> 
> 	* gcc.target/aarch64/pr108601.c: New test.
> ---
>  gcc/testsuite/gcc.target/aarch64/pr108601.c | 11 +++++
>  gcc/tree-vect-loop-manip.cc                 | 44 ++++++++++++++++++++
>  gcc/tree-vect-loop.cc                       | 46 ---------------------
>  gcc/tree-vectorizer.h                       |  3 --
>  4 files changed, 55 insertions(+), 49 deletions(-)
>  create mode 100644 gcc/testsuite/gcc.target/aarch64/pr108601.c
> 
> diff --git a/gcc/testsuite/gcc.target/aarch64/pr108601.c b/gcc/testsuite/gcc.target/aarch64/pr108601.c
> new file mode 100644
> index 00000000000..deb8b3061d8
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/aarch64/pr108601.c
> @@ -0,0 +1,11 @@
> +/* { dg-do compile } */
> +/* { dg-options "-O3 -fprofile-generate -mcpu=neoverse-v1" } */
> +
> +int
> +foo() {
> +  int flag = 1;
> +  for (; flag <= 1 << 21; flag <<= 1)
> +    ;
> +  return 0;
> +}
> +
> diff --git a/gcc/tree-vect-loop-manip.cc b/gcc/tree-vect-loop-manip.cc
> index b5c5f859144..c04fcf40c44 100644
> --- a/gcc/tree-vect-loop-manip.cc
> +++ b/gcc/tree-vect-loop-manip.cc
> @@ -1390,6 +1390,50 @@ iv_phi_p (stmt_vec_info stmt_info)
>    return true;
>  }
>  
> +/* Return true if vectorizer can peel for nonlinear iv.  */
> +static bool
> +vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
> +			      enum vect_induction_op_type induction_type)
> +{
> +  tree niters_skip;
> +  /* Init_expr will be update by vect_update_ivs_after_vectorizer,
> +     if niters or vf is unkown:
> +     For shift, when shift mount >= precision, there would be UD.
> +     For mult, don't known how to generate
> +     init_expr * pow (step, niters) for variable niters.
> +     For neg, it should be ok, since niters of vectorized main loop
> +     will always be multiple of 2.  */
> +  if ((!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
> +       || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant ())
> +      && induction_type != vect_step_op_neg)
> +    {
> +      if (dump_enabled_p ())
> +	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +			 "Peeling for epilogue is not supported"
> +			 " for nonlinear induction except neg"
> +			 " when iteration count is unknown.\n");
> +      return false;
> +    }
> +
> +  /* Also doens't support peel for neg when niter is variable.
> +     ??? generate something like niter_expr & 1 ? init_expr : -init_expr?  */
> +  niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
> +  if ((niters_skip != NULL_TREE
> +       && TREE_CODE (niters_skip) != INTEGER_CST)
> +      || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
> +	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
> +    {
> +      if (dump_enabled_p ())
> +	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> +			 "Peeling for alignement is not supported"
> +			 " for nonlinear induction when niters_skip"
> +			 " is not constant.\n");
> +      return false;
> +    }
> +
> +  return true;
> +}
> +
>  /* Function vect_can_advance_ivs_p
>  
>     In case the number of iterations that LOOP iterates is unknown at compile
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index f0801c23671..01b60a8de33 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -8808,49 +8808,6 @@ vect_update_nonlinear_iv (gimple_seq* stmts, tree vectype,
>  
>  }
>  
> -/* Return true if vectorizer can peel for nonlinear iv.  */
> -bool
> -vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
> -			      enum vect_induction_op_type induction_type)
> -{
> -  tree niters_skip;
> -  /* Init_expr will be update by vect_update_ivs_after_vectorizer,
> -     if niters is unkown:
> -     For shift, when shift mount >= precision, there would be UD.
> -     For mult, don't known how to generate
> -     init_expr * pow (step, niters) for variable niters.
> -     For neg, it should be ok, since niters of vectorized main loop
> -     will always be multiple of 2.  */
> -  if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
> -      && induction_type != vect_step_op_neg)
> -    {
> -      if (dump_enabled_p ())
> -	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -			 "Peeling for epilogue is not supported"
> -			 " for nonlinear induction except neg"
> -			 " when iteration count is unknown.\n");
> -      return false;
> -    }
> -
> -  /* Also doens't support peel for neg when niter is variable.
> -     ??? generate something like niter_expr & 1 ? init_expr : -init_expr?  */
> -  niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
> -  if ((niters_skip != NULL_TREE
> -       && TREE_CODE (niters_skip) != INTEGER_CST)
> -      || (!vect_use_loop_mask_for_alignment_p (loop_vinfo)
> -	  && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0))
> -    {
> -      if (dump_enabled_p ())
> -	dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> -			 "Peeling for alignement is not supported"
> -			 " for nonlinear induction when niters_skip"
> -			 " is not constant.\n");
> -      return false;
> -    }
> -
> -  return true;
> -}
> -
>  /* Function vectorizable_induction
>  
>     Check if STMT_INFO performs an nonlinear induction computation that can be
> @@ -8921,9 +8878,6 @@ vectorizable_nonlinear_induction (loop_vec_info loop_vinfo,
>        return false;
>      }
>  
> -  if (!vect_can_peel_nonlinear_iv_p (loop_vinfo, induction_type))
> -    return false;
> -
>    if (!INTEGRAL_TYPE_P (TREE_TYPE (vectype)))
>      {
>        if (dump_enabled_p ())
> diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
> index a2aa71bedc0..4ba653712e9 100644
> --- a/gcc/tree-vectorizer.h
> +++ b/gcc/tree-vectorizer.h
> @@ -2347,9 +2347,6 @@ extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree);
>  /* Nonlinear induction.  */
>  extern tree vect_peel_nonlinear_iv_init (gimple_seq*, tree, tree,
>  					 tree, enum vect_induction_op_type);
> -extern bool
> -vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo,
> -			      enum vect_induction_op_type induction_type);
>  
>  /* In tree-vect-slp.cc.  */
>  extern void vect_slp_init (void);
>