[V5] VECT: Support loop len control on EXTRACT_LAST vectorization
Checks
Commit Message
Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
powerpc and s390 with your suggestions:
- machine_mode len_load_mode = get_len_load_store_mode
- (loop_vinfo->vector_mode, true).require ();
- machine_mode len_store_mode = get_len_load_store_mode
- (loop_vinfo->vector_mode, false).require ();
+ machine_mode len_load_mode, len_store_mode;
+ if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
+ .exists (&len_load_mode))
+ return false;
+ if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
+ .exists (&len_store_mode))
+ return false;
Hi, @Kewen and @Stefan
Could you test this patch again ? Thanks.
Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
gcc/ChangeLog:
* tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
(vectorizable_live_operation): Add live vectorization for length loop control.
gcc/testsuite/ChangeLog:
* gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
* gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
---
.../riscv/rvv/autovec/partial/live-1.c | 34 +++++++
.../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++
gcc/tree-vect-loop.cc | 89 ++++++++++++++-----
3 files changed, 138 insertions(+), 20 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
Comments
On Mon, 21 Aug 2023, Juzhe-Zhong wrote:
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
> powerpc and s390 with your suggestions:
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
LGTM.
Richard.
> Hi, @Kewen and @Stefan
>
> Could you test this patch again ? Thanks.
>
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> gcc/ChangeLog:
>
> * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
> (vectorizable_live_operation): Add live vectorization for length loop control.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
> * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
>
> ---
> .../riscv/rvv/autovec/partial/live-1.c | 34 +++++++
> .../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++
> gcc/tree-vect-loop.cc | 89 ++++++++++++++-----
> 3 files changed, 138 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
>
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> new file mode 100644
> index 00000000000..75fa2eba8cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> @@ -0,0 +1,34 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
> +
> +#include <stdint-gcc.h>
> +
> +#define EXTRACT_LAST(TYPE) \
> + TYPE __attribute__ ((noinline, noclone)) \
> + test_##TYPE (TYPE *x, int n, TYPE value) \
> + { \
> + TYPE last; \
> + for (int j = 0; j < n; ++j) \
> + { \
> + last = x[j]; \
> + x[j] = last * value; \
> + } \
> + return last; \
> + }
> +
> +#define TEST_ALL(T) \
> + T (int8_t) \
> + T (int16_t) \
> + T (int32_t) \
> + T (int64_t) \
> + T (uint8_t) \
> + T (uint16_t) \
> + T (uint32_t) \
> + T (uint64_t) \
> + T (_Float16) \
> + T (float) \
> + T (double)
> +
> +TEST_ALL (EXTRACT_LAST)
> +
> +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> new file mode 100644
> index 00000000000..42913a112c6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> @@ -0,0 +1,35 @@
> +/* { dg-do run { target { riscv_vector } } } */
> +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
> +
> +#include "live-1.c"
> +
> +#define N 107
> +#define OP 70
> +
> +#define TEST_LOOP(TYPE) \
> + { \
> + TYPE a[N]; \
> + for (int i = 0; i < N; ++i) \
> + { \
> + a[i] = i * 2 + (i % 3); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + TYPE expected = a[N - 1]; \
> + TYPE res = test_##TYPE (a, N, OP); \
> + if (res != expected) \
> + __builtin_abort (); \
> + for (int i = 0; i < N; ++i) \
> + { \
> + TYPE old = i * 2 + (i % 3); \
> + if (a[i] != (TYPE) (old * (TYPE) OP)) \
> + __builtin_abort (); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + }
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> + TEST_ALL (TEST_LOOP);
> + return 0;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 1fcd8d07ea1..1cd6c291377 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
> #include "tree-pass.h"
> #include "ssa.h"
> #include "optabs-tree.h"
> +#include "memmodel.h"
> +#include "optabs.h"
> #include "diagnostic-core.h"
> #include "fold-const.h"
> #include "stor-layout.h"
> @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
> if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
> return false;
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> signed char partial_load_bias = internal_len_load_store_bias
> (IFN_LEN_LOAD, len_load_mode);
> @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> /* No transformation required. */
> if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
> {
> - if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> - OPTIMIZE_FOR_SPEED))
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "can't operate on partial vectors "
> - "because the target doesn't support extract "
> - "last reduction.\n");
> - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> - }
> - else if (slp_node)
> + if (slp_node)
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> else
> {
> gcc_assert (ncopies == 1 && !slp_node);
> - vect_record_loop_mask (loop_vinfo,
> - &LOOP_VINFO_MASKS (loop_vinfo),
> - 1, vectype, NULL);
> + if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> + OPTIMIZE_FOR_SPEED))
> + vect_record_loop_mask (loop_vinfo,
> + &LOOP_VINFO_MASKS (loop_vinfo),
> + 1, vectype, NULL);
> + else if (can_vec_extract_var_idx_p (
> + TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
> + vect_record_loop_len (loop_vinfo,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 1);
> + else
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (
> + MSG_MISSED_OPTIMIZATION, vect_location,
> + "can't operate on partial vectors "
> + "because the target doesn't support extract "
> + "last reduction.\n");
> + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> + }
> }
> }
> /* ??? Enable for loop costing as well. */
> @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> gimple *vec_stmt;
> if (slp_node)
> {
> - gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
> + gcc_assert (!loop_vinfo
> + || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
> + && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
>
> /* Get the correct slp vectorized stmt. */
> vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
> @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>
> gimple_seq stmts = NULL;
> tree new_tree;
> - if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> + {
> + /* Emit:
> +
> + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> +
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (ncopies == 1 && !slp_node);
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree len
> + = vect_get_loop_len (loop_vinfo, &gsi,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 0, 0);
> +
> + /* BIAS - 1. */
> + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + tree bias_minus_one
> + = int_const_binop (MINUS_EXPR,
> + build_int_cst (TREE_TYPE (len), biasval),
> + build_one_cst (TREE_TYPE (len)));
> +
> + /* LAST_INDEX = LEN + (BIAS - 1). */
> + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> + len, bias_minus_one);
> +
> + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> + tree scalar_res
> + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> + vec_lhs_phi, last_index);
> +
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> {
> /* Emit:
>
>
Hi Juzhe,
on 2023/8/21 18:59, Juzhe-Zhong wrote:
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
> powerpc and s390 with your suggestions:
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> Hi, @Kewen and @Stefan
>
> Could you test this patch again ? Thanks.
I confirmed it's bootstrapped and regress-tested on
powerpc64le-linux-gnu P9/P10. Thanks!
BR,
Kewen
>
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> gcc/ChangeLog:
>
> * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
> (vectorizable_live_operation): Add live vectorization for length loop control.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
> * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
>
> ---
> .../riscv/rvv/autovec/partial/live-1.c | 34 +++++++
> .../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++
> gcc/tree-vect-loop.cc | 89 ++++++++++++++-----
> 3 files changed, 138 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
>
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> new file mode 100644
> index 00000000000..75fa2eba8cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> @@ -0,0 +1,34 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
> +
> +#include <stdint-gcc.h>
> +
> +#define EXTRACT_LAST(TYPE) \
> + TYPE __attribute__ ((noinline, noclone)) \
> + test_##TYPE (TYPE *x, int n, TYPE value) \
> + { \
> + TYPE last; \
> + for (int j = 0; j < n; ++j) \
> + { \
> + last = x[j]; \
> + x[j] = last * value; \
> + } \
> + return last; \
> + }
> +
> +#define TEST_ALL(T) \
> + T (int8_t) \
> + T (int16_t) \
> + T (int32_t) \
> + T (int64_t) \
> + T (uint8_t) \
> + T (uint16_t) \
> + T (uint32_t) \
> + T (uint64_t) \
> + T (_Float16) \
> + T (float) \
> + T (double)
> +
> +TEST_ALL (EXTRACT_LAST)
> +
> +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> new file mode 100644
> index 00000000000..42913a112c6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> @@ -0,0 +1,35 @@
> +/* { dg-do run { target { riscv_vector } } } */
> +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
> +
> +#include "live-1.c"
> +
> +#define N 107
> +#define OP 70
> +
> +#define TEST_LOOP(TYPE) \
> + { \
> + TYPE a[N]; \
> + for (int i = 0; i < N; ++i) \
> + { \
> + a[i] = i * 2 + (i % 3); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + TYPE expected = a[N - 1]; \
> + TYPE res = test_##TYPE (a, N, OP); \
> + if (res != expected) \
> + __builtin_abort (); \
> + for (int i = 0; i < N; ++i) \
> + { \
> + TYPE old = i * 2 + (i % 3); \
> + if (a[i] != (TYPE) (old * (TYPE) OP)) \
> + __builtin_abort (); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + }
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> + TEST_ALL (TEST_LOOP);
> + return 0;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 1fcd8d07ea1..1cd6c291377 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
> #include "tree-pass.h"
> #include "ssa.h"
> #include "optabs-tree.h"
> +#include "memmodel.h"
> +#include "optabs.h"
> #include "diagnostic-core.h"
> #include "fold-const.h"
> #include "stor-layout.h"
> @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
> if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
> return false;
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> signed char partial_load_bias = internal_len_load_store_bias
> (IFN_LEN_LOAD, len_load_mode);
> @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> /* No transformation required. */
> if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
> {
> - if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> - OPTIMIZE_FOR_SPEED))
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "can't operate on partial vectors "
> - "because the target doesn't support extract "
> - "last reduction.\n");
> - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> - }
> - else if (slp_node)
> + if (slp_node)
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> else
> {
> gcc_assert (ncopies == 1 && !slp_node);
> - vect_record_loop_mask (loop_vinfo,
> - &LOOP_VINFO_MASKS (loop_vinfo),
> - 1, vectype, NULL);
> + if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> + OPTIMIZE_FOR_SPEED))
> + vect_record_loop_mask (loop_vinfo,
> + &LOOP_VINFO_MASKS (loop_vinfo),
> + 1, vectype, NULL);
> + else if (can_vec_extract_var_idx_p (
> + TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
> + vect_record_loop_len (loop_vinfo,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 1);
> + else
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (
> + MSG_MISSED_OPTIMIZATION, vect_location,
> + "can't operate on partial vectors "
> + "because the target doesn't support extract "
> + "last reduction.\n");
> + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> + }
> }
> }
> /* ??? Enable for loop costing as well. */
> @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> gimple *vec_stmt;
> if (slp_node)
> {
> - gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
> + gcc_assert (!loop_vinfo
> + || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
> + && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
>
> /* Get the correct slp vectorized stmt. */
> vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
> @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>
> gimple_seq stmts = NULL;
> tree new_tree;
> - if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> + {
> + /* Emit:
> +
> + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> +
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (ncopies == 1 && !slp_node);
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree len
> + = vect_get_loop_len (loop_vinfo, &gsi,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 0, 0);
> +
> + /* BIAS - 1. */
> + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + tree bias_minus_one
> + = int_const_binop (MINUS_EXPR,
> + build_int_cst (TREE_TYPE (len), biasval),
> + build_one_cst (TREE_TYPE (len)));
> +
> + /* LAST_INDEX = LEN + (BIAS - 1). */
> + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> + len, bias_minus_one);
> +
> + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> + tree scalar_res
> + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> + vec_lhs_phi, last_index);
> +
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> {
> /* Emit:
>
Thanks kewen.
I am gonna wait for stefan's test result before I merge this patch.
Thanks.
juzhe.zhong@rivai.ai
From: Kewen.Lin
Date: 2023-08-22 09:12
To: Juzhe-Zhong
CC: richard.sandiford; rguenther; stefansf; gcc-patches
Subject: Re: [PATCH V5] VECT: Support loop len control on EXTRACT_LAST vectorization
Hi Juzhe,
on 2023/8/21 18:59, Juzhe-Zhong wrote:
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
> powerpc and s390 with your suggestions:
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> Hi, @Kewen and @Stefan
>
> Could you test this patch again ? Thanks.
I confirmed it's bootstrapped and regress-tested on
powerpc64le-linux-gnu P9/P10. Thanks!
BR,
Kewen
>
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> gcc/ChangeLog:
>
> * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
> (vectorizable_live_operation): Add live vectorization for length loop control.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
> * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
>
> ---
> .../riscv/rvv/autovec/partial/live-1.c | 34 +++++++
> .../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++
> gcc/tree-vect-loop.cc | 89 ++++++++++++++-----
> 3 files changed, 138 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
>
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> new file mode 100644
> index 00000000000..75fa2eba8cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> @@ -0,0 +1,34 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
> +
> +#include <stdint-gcc.h>
> +
> +#define EXTRACT_LAST(TYPE) \
> + TYPE __attribute__ ((noinline, noclone)) \
> + test_##TYPE (TYPE *x, int n, TYPE value) \
> + { \
> + TYPE last; \
> + for (int j = 0; j < n; ++j) \
> + { \
> + last = x[j]; \
> + x[j] = last * value; \
> + } \
> + return last; \
> + }
> +
> +#define TEST_ALL(T) \
> + T (int8_t) \
> + T (int16_t) \
> + T (int32_t) \
> + T (int64_t) \
> + T (uint8_t) \
> + T (uint16_t) \
> + T (uint32_t) \
> + T (uint64_t) \
> + T (_Float16) \
> + T (float) \
> + T (double)
> +
> +TEST_ALL (EXTRACT_LAST)
> +
> +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> new file mode 100644
> index 00000000000..42913a112c6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> @@ -0,0 +1,35 @@
> +/* { dg-do run { target { riscv_vector } } } */
> +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
> +
> +#include "live-1.c"
> +
> +#define N 107
> +#define OP 70
> +
> +#define TEST_LOOP(TYPE) \
> + { \
> + TYPE a[N]; \
> + for (int i = 0; i < N; ++i) \
> + { \
> + a[i] = i * 2 + (i % 3); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + TYPE expected = a[N - 1]; \
> + TYPE res = test_##TYPE (a, N, OP); \
> + if (res != expected) \
> + __builtin_abort (); \
> + for (int i = 0; i < N; ++i) \
> + { \
> + TYPE old = i * 2 + (i % 3); \
> + if (a[i] != (TYPE) (old * (TYPE) OP)) \
> + __builtin_abort (); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + }
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> + TEST_ALL (TEST_LOOP);
> + return 0;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 1fcd8d07ea1..1cd6c291377 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
> #include "tree-pass.h"
> #include "ssa.h"
> #include "optabs-tree.h"
> +#include "memmodel.h"
> +#include "optabs.h"
> #include "diagnostic-core.h"
> #include "fold-const.h"
> #include "stor-layout.h"
> @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
> if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
> return false;
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> signed char partial_load_bias = internal_len_load_store_bias
> (IFN_LEN_LOAD, len_load_mode);
> @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> /* No transformation required. */
> if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
> {
> - if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> - OPTIMIZE_FOR_SPEED))
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "can't operate on partial vectors "
> - "because the target doesn't support extract "
> - "last reduction.\n");
> - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> - }
> - else if (slp_node)
> + if (slp_node)
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> else
> {
> gcc_assert (ncopies == 1 && !slp_node);
> - vect_record_loop_mask (loop_vinfo,
> - &LOOP_VINFO_MASKS (loop_vinfo),
> - 1, vectype, NULL);
> + if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> + OPTIMIZE_FOR_SPEED))
> + vect_record_loop_mask (loop_vinfo,
> + &LOOP_VINFO_MASKS (loop_vinfo),
> + 1, vectype, NULL);
> + else if (can_vec_extract_var_idx_p (
> + TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
> + vect_record_loop_len (loop_vinfo,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 1);
> + else
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (
> + MSG_MISSED_OPTIMIZATION, vect_location,
> + "can't operate on partial vectors "
> + "because the target doesn't support extract "
> + "last reduction.\n");
> + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> + }
> }
> }
> /* ??? Enable for loop costing as well. */
> @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> gimple *vec_stmt;
> if (slp_node)
> {
> - gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
> + gcc_assert (!loop_vinfo
> + || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
> + && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
>
> /* Get the correct slp vectorized stmt. */
> vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
> @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>
> gimple_seq stmts = NULL;
> tree new_tree;
> - if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> + {
> + /* Emit:
> +
> + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> +
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (ncopies == 1 && !slp_node);
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree len
> + = vect_get_loop_len (loop_vinfo, &gsi,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 0, 0);
> +
> + /* BIAS - 1. */
> + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + tree bias_minus_one
> + = int_const_binop (MINUS_EXPR,
> + build_int_cst (TREE_TYPE (len), biasval),
> + build_one_cst (TREE_TYPE (len)));
> +
> + /* LAST_INDEX = LEN + (BIAS - 1). */
> + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> + len, bias_minus_one);
> +
> + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> + tree scalar_res
> + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> + vec_lhs_phi, last_index);
> +
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> {
> /* Emit:
>
On Mon, Aug 21, 2023 at 06:59:55PM +0800, Juzhe-Zhong wrote:
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
> powerpc and s390 with your suggestions:
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> Hi, @Kewen and @Stefan
Successfully bootstrapped and regtested on s390 (z900 as well as z16;
both for 31- and 64-bit).
Thanks,
Stefan
>
> Could you test this patch again ? Thanks.
>
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> gcc/ChangeLog:
>
> * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
> (vectorizable_live_operation): Add live vectorization for length loop control.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
> * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
>
> ---
> .../riscv/rvv/autovec/partial/live-1.c | 34 +++++++
> .../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++
> gcc/tree-vect-loop.cc | 89 ++++++++++++++-----
> 3 files changed, 138 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
>
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> new file mode 100644
> index 00000000000..75fa2eba8cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> @@ -0,0 +1,34 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
> +
> +#include <stdint-gcc.h>
> +
> +#define EXTRACT_LAST(TYPE) \
> + TYPE __attribute__ ((noinline, noclone)) \
> + test_##TYPE (TYPE *x, int n, TYPE value) \
> + { \
> + TYPE last; \
> + for (int j = 0; j < n; ++j) \
> + { \
> + last = x[j]; \
> + x[j] = last * value; \
> + } \
> + return last; \
> + }
> +
> +#define TEST_ALL(T) \
> + T (int8_t) \
> + T (int16_t) \
> + T (int32_t) \
> + T (int64_t) \
> + T (uint8_t) \
> + T (uint16_t) \
> + T (uint32_t) \
> + T (uint64_t) \
> + T (_Float16) \
> + T (float) \
> + T (double)
> +
> +TEST_ALL (EXTRACT_LAST)
> +
> +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> new file mode 100644
> index 00000000000..42913a112c6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> @@ -0,0 +1,35 @@
> +/* { dg-do run { target { riscv_vector } } } */
> +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
> +
> +#include "live-1.c"
> +
> +#define N 107
> +#define OP 70
> +
> +#define TEST_LOOP(TYPE) \
> + { \
> + TYPE a[N]; \
> + for (int i = 0; i < N; ++i) \
> + { \
> + a[i] = i * 2 + (i % 3); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + TYPE expected = a[N - 1]; \
> + TYPE res = test_##TYPE (a, N, OP); \
> + if (res != expected) \
> + __builtin_abort (); \
> + for (int i = 0; i < N; ++i) \
> + { \
> + TYPE old = i * 2 + (i % 3); \
> + if (a[i] != (TYPE) (old * (TYPE) OP)) \
> + __builtin_abort (); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + }
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> + TEST_ALL (TEST_LOOP);
> + return 0;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 1fcd8d07ea1..1cd6c291377 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
> #include "tree-pass.h"
> #include "ssa.h"
> #include "optabs-tree.h"
> +#include "memmodel.h"
> +#include "optabs.h"
> #include "diagnostic-core.h"
> #include "fold-const.h"
> #include "stor-layout.h"
> @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
> if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
> return false;
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> signed char partial_load_bias = internal_len_load_store_bias
> (IFN_LEN_LOAD, len_load_mode);
> @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> /* No transformation required. */
> if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
> {
> - if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> - OPTIMIZE_FOR_SPEED))
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "can't operate on partial vectors "
> - "because the target doesn't support extract "
> - "last reduction.\n");
> - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> - }
> - else if (slp_node)
> + if (slp_node)
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> else
> {
> gcc_assert (ncopies == 1 && !slp_node);
> - vect_record_loop_mask (loop_vinfo,
> - &LOOP_VINFO_MASKS (loop_vinfo),
> - 1, vectype, NULL);
> + if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> + OPTIMIZE_FOR_SPEED))
> + vect_record_loop_mask (loop_vinfo,
> + &LOOP_VINFO_MASKS (loop_vinfo),
> + 1, vectype, NULL);
> + else if (can_vec_extract_var_idx_p (
> + TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
> + vect_record_loop_len (loop_vinfo,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 1);
> + else
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (
> + MSG_MISSED_OPTIMIZATION, vect_location,
> + "can't operate on partial vectors "
> + "because the target doesn't support extract "
> + "last reduction.\n");
> + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> + }
> }
> }
> /* ??? Enable for loop costing as well. */
> @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> gimple *vec_stmt;
> if (slp_node)
> {
> - gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
> + gcc_assert (!loop_vinfo
> + || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
> + && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
>
> /* Get the correct slp vectorized stmt. */
> vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
> @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>
> gimple_seq stmts = NULL;
> tree new_tree;
> - if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> + {
> + /* Emit:
> +
> + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> +
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (ncopies == 1 && !slp_node);
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree len
> + = vect_get_loop_len (loop_vinfo, &gsi,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 0, 0);
> +
> + /* BIAS - 1. */
> + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + tree bias_minus_one
> + = int_const_binop (MINUS_EXPR,
> + build_int_cst (TREE_TYPE (len), biasval),
> + build_one_cst (TREE_TYPE (len)));
> +
> + /* LAST_INDEX = LEN + (BIAS - 1). */
> + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> + len, bias_minus_one);
> +
> + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> + tree scalar_res
> + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> + vec_lhs_phi, last_index);
> +
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> {
> /* Emit:
>
> --
> 2.36.3
>
Committed, thanks all.
Pan
-----Original Message-----
From: Gcc-patches <gcc-patches-bounces+pan2.li=intel.com@gcc.gnu.org> On Behalf Of Stefan Schulze Frielinghaus via Gcc-patches
Sent: Tuesday, August 22, 2023 2:23 PM
To: Juzhe-Zhong <juzhe.zhong@rivai.ai>
Cc: gcc-patches@gcc.gnu.org; richard.sandiford@arm.com; rguenther@suse.de; linkw@linux.ibm.com
Subject: Re: [PATCH V5] VECT: Support loop len control on EXTRACT_LAST vectorization
On Mon, Aug 21, 2023 at 06:59:55PM +0800, Juzhe-Zhong wrote:
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> Hi, @Richi and @Richard, base on previous disscussion, I simpily fix issuses for
> powerpc and s390 with your suggestions:
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> Hi, @Kewen and @Stefan
Successfully bootstrapped and regtested on s390 (z900 as well as z16;
both for 31- and 64-bit).
Thanks,
Stefan
>
> Could you test this patch again ? Thanks.
>
> Co-Authored-By: Kewen.Lin <linkw@linux.ibm.com>
>
> gcc/ChangeLog:
>
> * tree-vect-loop.cc (vect_verify_loop_lens): Add exists check.
> (vectorizable_live_operation): Add live vectorization for length loop control.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/riscv/rvv/autovec/partial/live-1.c: New test.
> * gcc.target/riscv/rvv/autovec/partial/live_run-1.c: New test.
>
> ---
> .../riscv/rvv/autovec/partial/live-1.c | 34 +++++++
> .../riscv/rvv/autovec/partial/live_run-1.c | 35 ++++++++
> gcc/tree-vect-loop.cc | 89 ++++++++++++++-----
> 3 files changed, 138 insertions(+), 20 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> create mode 100644 gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
>
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> new file mode 100644
> index 00000000000..75fa2eba8cc
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live-1.c
> @@ -0,0 +1,34 @@
> +/* { dg-do compile } */
> +/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
> +
> +#include <stdint-gcc.h>
> +
> +#define EXTRACT_LAST(TYPE) \
> + TYPE __attribute__ ((noinline, noclone)) \
> + test_##TYPE (TYPE *x, int n, TYPE value) \
> + { \
> + TYPE last; \
> + for (int j = 0; j < n; ++j) \
> + { \
> + last = x[j]; \
> + x[j] = last * value; \
> + } \
> + return last; \
> + }
> +
> +#define TEST_ALL(T) \
> + T (int8_t) \
> + T (int16_t) \
> + T (int32_t) \
> + T (int64_t) \
> + T (uint8_t) \
> + T (uint16_t) \
> + T (uint32_t) \
> + T (uint64_t) \
> + T (_Float16) \
> + T (float) \
> + T (double)
> +
> +TEST_ALL (EXTRACT_LAST)
> +
> +/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
> diff --git a/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> new file mode 100644
> index 00000000000..42913a112c6
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/riscv/rvv/autovec/partial/live_run-1.c
> @@ -0,0 +1,35 @@
> +/* { dg-do run { target { riscv_vector } } } */
> +/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
> +
> +#include "live-1.c"
> +
> +#define N 107
> +#define OP 70
> +
> +#define TEST_LOOP(TYPE) \
> + { \
> + TYPE a[N]; \
> + for (int i = 0; i < N; ++i) \
> + { \
> + a[i] = i * 2 + (i % 3); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + TYPE expected = a[N - 1]; \
> + TYPE res = test_##TYPE (a, N, OP); \
> + if (res != expected) \
> + __builtin_abort (); \
> + for (int i = 0; i < N; ++i) \
> + { \
> + TYPE old = i * 2 + (i % 3); \
> + if (a[i] != (TYPE) (old * (TYPE) OP)) \
> + __builtin_abort (); \
> + asm volatile ("" ::: "memory"); \
> + } \
> + }
> +
> +int __attribute__ ((optimize (1)))
> +main (void)
> +{
> + TEST_ALL (TEST_LOOP);
> + return 0;
> +}
> diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc
> index 1fcd8d07ea1..1cd6c291377 100644
> --- a/gcc/tree-vect-loop.cc
> +++ b/gcc/tree-vect-loop.cc
> @@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
> #include "tree-pass.h"
> #include "ssa.h"
> #include "optabs-tree.h"
> +#include "memmodel.h"
> +#include "optabs.h"
> #include "diagnostic-core.h"
> #include "fold-const.h"
> #include "stor-layout.h"
> @@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
> if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
> return false;
>
> - machine_mode len_load_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, true).require ();
> - machine_mode len_store_mode = get_len_load_store_mode
> - (loop_vinfo->vector_mode, false).require ();
> + machine_mode len_load_mode, len_store_mode;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
> + .exists (&len_load_mode))
> + return false;
> + if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
> + .exists (&len_store_mode))
> + return false;
>
> signed char partial_load_bias = internal_len_load_store_bias
> (IFN_LEN_LOAD, len_load_mode);
> @@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> /* No transformation required. */
> if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
> {
> - if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> - OPTIMIZE_FOR_SPEED))
> - {
> - if (dump_enabled_p ())
> - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> - "can't operate on partial vectors "
> - "because the target doesn't support extract "
> - "last reduction.\n");
> - LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> - }
> - else if (slp_node)
> + if (slp_node)
> {
> if (dump_enabled_p ())
> dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
> @@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> else
> {
> gcc_assert (ncopies == 1 && !slp_node);
> - vect_record_loop_mask (loop_vinfo,
> - &LOOP_VINFO_MASKS (loop_vinfo),
> - 1, vectype, NULL);
> + if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
> + OPTIMIZE_FOR_SPEED))
> + vect_record_loop_mask (loop_vinfo,
> + &LOOP_VINFO_MASKS (loop_vinfo),
> + 1, vectype, NULL);
> + else if (can_vec_extract_var_idx_p (
> + TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
> + vect_record_loop_len (loop_vinfo,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 1);
> + else
> + {
> + if (dump_enabled_p ())
> + dump_printf_loc (
> + MSG_MISSED_OPTIMIZATION, vect_location,
> + "can't operate on partial vectors "
> + "because the target doesn't support extract "
> + "last reduction.\n");
> + LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
> + }
> }
> }
> /* ??? Enable for loop costing as well. */
> @@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
> gimple *vec_stmt;
> if (slp_node)
> {
> - gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
> + gcc_assert (!loop_vinfo
> + || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
> + && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
>
> /* Get the correct slp vectorized stmt. */
> vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
> @@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
>
> gimple_seq stmts = NULL;
> tree new_tree;
> - if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
> + {
> + /* Emit:
> +
> + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
> +
> + where VEC_LHS is the vectorized live-out result and MASK is
> + the loop mask for the final iteration. */
> + gcc_assert (ncopies == 1 && !slp_node);
> + gimple_seq tem = NULL;
> + gimple_stmt_iterator gsi = gsi_last (tem);
> + tree len
> + = vect_get_loop_len (loop_vinfo, &gsi,
> + &LOOP_VINFO_LENS (loop_vinfo),
> + 1, vectype, 0, 0);
> +
> + /* BIAS - 1. */
> + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
> + tree bias_minus_one
> + = int_const_binop (MINUS_EXPR,
> + build_int_cst (TREE_TYPE (len), biasval),
> + build_one_cst (TREE_TYPE (len)));
> +
> + /* LAST_INDEX = LEN + (BIAS - 1). */
> + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
> + len, bias_minus_one);
> +
> + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
> + tree scalar_res
> + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
> + vec_lhs_phi, last_index);
> +
> + /* Convert the extracted vector element to the scalar type. */
> + new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
> + }
> + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
> {
> /* Emit:
>
> --
> 2.36.3
>
new file mode 100644
@@ -0,0 +1,34 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-march=rv32gcv -mabi=ilp32d --param riscv-autovec-preference=scalable -fdump-tree-optimized-details" } */
+
+#include <stdint-gcc.h>
+
+#define EXTRACT_LAST(TYPE) \
+ TYPE __attribute__ ((noinline, noclone)) \
+ test_##TYPE (TYPE *x, int n, TYPE value) \
+ { \
+ TYPE last; \
+ for (int j = 0; j < n; ++j) \
+ { \
+ last = x[j]; \
+ x[j] = last * value; \
+ } \
+ return last; \
+ }
+
+#define TEST_ALL(T) \
+ T (int8_t) \
+ T (int16_t) \
+ T (int32_t) \
+ T (int64_t) \
+ T (uint8_t) \
+ T (uint16_t) \
+ T (uint32_t) \
+ T (uint64_t) \
+ T (_Float16) \
+ T (float) \
+ T (double)
+
+TEST_ALL (EXTRACT_LAST)
+
+/* { dg-final { scan-tree-dump-times "\.VEC_EXTRACT" 10 "optimized" } } */
new file mode 100644
@@ -0,0 +1,35 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-additional-options "--param riscv-autovec-preference=scalable" } */
+
+#include "live-1.c"
+
+#define N 107
+#define OP 70
+
+#define TEST_LOOP(TYPE) \
+ { \
+ TYPE a[N]; \
+ for (int i = 0; i < N; ++i) \
+ { \
+ a[i] = i * 2 + (i % 3); \
+ asm volatile ("" ::: "memory"); \
+ } \
+ TYPE expected = a[N - 1]; \
+ TYPE res = test_##TYPE (a, N, OP); \
+ if (res != expected) \
+ __builtin_abort (); \
+ for (int i = 0; i < N; ++i) \
+ { \
+ TYPE old = i * 2 + (i % 3); \
+ if (a[i] != (TYPE) (old * (TYPE) OP)) \
+ __builtin_abort (); \
+ asm volatile ("" ::: "memory"); \
+ } \
+ }
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ TEST_ALL (TEST_LOOP);
+ return 0;
+}
@@ -32,6 +32,8 @@ along with GCC; see the file COPYING3. If not see
#include "tree-pass.h"
#include "ssa.h"
#include "optabs-tree.h"
+#include "memmodel.h"
+#include "optabs.h"
#include "diagnostic-core.h"
#include "fold-const.h"
#include "stor-layout.h"
@@ -1465,10 +1467,13 @@ vect_verify_loop_lens (loop_vec_info loop_vinfo)
if (LOOP_VINFO_LENS (loop_vinfo).is_empty ())
return false;
- machine_mode len_load_mode = get_len_load_store_mode
- (loop_vinfo->vector_mode, true).require ();
- machine_mode len_store_mode = get_len_load_store_mode
- (loop_vinfo->vector_mode, false).require ();
+ machine_mode len_load_mode, len_store_mode;
+ if (!get_len_load_store_mode (loop_vinfo->vector_mode, true)
+ .exists (&len_load_mode))
+ return false;
+ if (!get_len_load_store_mode (loop_vinfo->vector_mode, false)
+ .exists (&len_store_mode))
+ return false;
signed char partial_load_bias = internal_len_load_store_bias
(IFN_LEN_LOAD, len_load_mode);
@@ -10301,17 +10306,7 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
/* No transformation required. */
if (loop_vinfo && LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo))
{
- if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
- OPTIMIZE_FOR_SPEED))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "can't operate on partial vectors "
- "because the target doesn't support extract "
- "last reduction.\n");
- LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
- }
- else if (slp_node)
+ if (slp_node)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
@@ -10331,9 +10326,26 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
else
{
gcc_assert (ncopies == 1 && !slp_node);
- vect_record_loop_mask (loop_vinfo,
- &LOOP_VINFO_MASKS (loop_vinfo),
- 1, vectype, NULL);
+ if (direct_internal_fn_supported_p (IFN_EXTRACT_LAST, vectype,
+ OPTIMIZE_FOR_SPEED))
+ vect_record_loop_mask (loop_vinfo,
+ &LOOP_VINFO_MASKS (loop_vinfo),
+ 1, vectype, NULL);
+ else if (can_vec_extract_var_idx_p (
+ TYPE_MODE (vectype), TYPE_MODE (TREE_TYPE (vectype))))
+ vect_record_loop_len (loop_vinfo,
+ &LOOP_VINFO_LENS (loop_vinfo),
+ 1, vectype, 1);
+ else
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (
+ MSG_MISSED_OPTIMIZATION, vect_location,
+ "can't operate on partial vectors "
+ "because the target doesn't support extract "
+ "last reduction.\n");
+ LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P (loop_vinfo) = false;
+ }
}
}
/* ??? Enable for loop costing as well. */
@@ -10359,7 +10371,9 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
gimple *vec_stmt;
if (slp_node)
{
- gcc_assert (!loop_vinfo || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
+ gcc_assert (!loop_vinfo
+ || (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+ && !LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)));
/* Get the correct slp vectorized stmt. */
vec_lhs = SLP_TREE_VEC_DEFS (slp_node)[vec_entry];
@@ -10403,7 +10417,42 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info,
gimple_seq stmts = NULL;
tree new_tree;
- if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
+ if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
+ {
+ /* Emit:
+
+ SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>
+
+ where VEC_LHS is the vectorized live-out result and MASK is
+ the loop mask for the final iteration. */
+ gcc_assert (ncopies == 1 && !slp_node);
+ gimple_seq tem = NULL;
+ gimple_stmt_iterator gsi = gsi_last (tem);
+ tree len
+ = vect_get_loop_len (loop_vinfo, &gsi,
+ &LOOP_VINFO_LENS (loop_vinfo),
+ 1, vectype, 0, 0);
+
+ /* BIAS - 1. */
+ signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo);
+ tree bias_minus_one
+ = int_const_binop (MINUS_EXPR,
+ build_int_cst (TREE_TYPE (len), biasval),
+ build_one_cst (TREE_TYPE (len)));
+
+ /* LAST_INDEX = LEN + (BIAS - 1). */
+ tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len),
+ len, bias_minus_one);
+
+ /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */
+ tree scalar_res
+ = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype),
+ vec_lhs_phi, last_index);
+
+ /* Convert the extracted vector element to the scalar type. */
+ new_tree = gimple_convert (&stmts, lhs_type, scalar_res);
+ }
+ else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo))
{
/* Emit: