[V2] VECT: Enhance SLP of MASK_LEN_GATHER_LOAD[PR111721]
Checks
Commit Message
This patch fixes this following FAILs in RISC-V regression:
FAIL: gcc.dg/vect/vect-gather-1.c -flto -ffat-lto-objects scan-tree-dump vect "Loop contains only SLP stmts"
FAIL: gcc.dg/vect/vect-gather-1.c scan-tree-dump vect "Loop contains only SLP stmts"
FAIL: gcc.dg/vect/vect-gather-3.c -flto -ffat-lto-objects scan-tree-dump vect "Loop contains only SLP stmts"
FAIL: gcc.dg/vect/vect-gather-3.c scan-tree-dump vect "Loop contains only SLP stmts"
The root cause of these FAIL is that GCC SLP failed on MASK_LEN_GATHER_LOAD.
To naturally reuse the current flow of GATHER_LOAD/MASK_GATHER_LOAD.
I adjust MASK_LEN_GATHER_LOAD/MASK_LEN_SCATTER_STORE pattern in tree-vect-patterns.cc
Here is adjustment in tree-vect-patterns.cc:
1. For un-conditional gather load/scatter store:
MASK_LEN_GATHER_LOAD (base, offset, scale, zero, -1) ---> MASK_LEN_GATHER_LOAD (base, offset, scale, zero)
Note that we remove the dummy mask (-1) of MASK_LEN_GATHER_LOAD, so that we can reuse the current SLP flow of GATHER_LOAD.
2. For conditional gather load/scatter store:
We don't change the IR, so they have an additional conditional mask. Then, we reuse the current flow of MASK_GATHER_LOAD.
So, after the recognization of patterns (tree-vect-patterns.cc), we will end up with scalar gather/scatter IR with different
num arguments. (4 arguments for un-conditional, 5 arguments for conditional).
The difference only apply on scalar gather/scatter IR. Pass through "call" argument to "internal_fn_mask_index" and return
the mask_index according to CALL for mask_len_gather/mask_len_scatter.
For vector IR, they are always same (keep original format): MASK_GATHER_LOAD (ptr, offset, scale, zero, mask, len, bias).
Hence, the optab of mask_len gather/scatter don't change.
To conclude, we only change the format of mask_len gather/scatter scalar IR in tree-vect-patterns.cc
It seems the flow of MASK_LEN_GATHER_LOAD/MASK_LEN_SCATTER_STORE after this patch seems to be more natural and reasonable.
Also, I realize that SLP of conditional gather_load is missing so I append a test for that.
RISC-V regression passed and Bootstrap && Regression on X86 passed.
Ok for trunk ?
gcc/ChangeLog:
* internal-fn.cc (internal_fn_mask_index): Add call argument.
* internal-fn.h (internal_fn_mask_index): Ditto.
* tree-vect-patterns.cc (vect_recog_gather_scatter_pattern): Delete MASK_LEN_GATHER_LOAD/MASK_LEN_SCATTER_STORE.
* tree-vect-slp.cc (vect_get_operand_map): Add MASK_LEN_GATHER_LOAD.
(vect_build_slp_tree_1): Ditto.
(vect_build_slp_tree_2): Ditto.
* tree-vect-stmts.cc (exist_non_indexing_operands_for_use_p): Ditto.
(vectorizable_store): Adapt for new interface of internal_fn_mask_index.
(vectorizable_load): Ditto.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-gather-6.c: New test.
---
gcc/internal-fn.cc | 16 ++++++++++++++--
gcc/internal-fn.h | 2 +-
gcc/testsuite/gcc.dg/vect/vect-gather-6.c | 15 +++++++++++++++
gcc/tree-vect-patterns.cc | 4 +---
gcc/tree-vect-slp.cc | 17 +++++++++++++++--
gcc/tree-vect-stmts.cc | 6 +++---
6 files changed, 49 insertions(+), 11 deletions(-)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-gather-6.c
@@ -4701,7 +4701,7 @@ internal_fn_len_index (internal_fn fn)
otherwise return -1. */
int
-internal_fn_mask_index (internal_fn fn)
+internal_fn_mask_index (internal_fn fn, gcall *call)
{
switch (fn)
{
@@ -4717,9 +4717,21 @@ internal_fn_mask_index (internal_fn fn)
case IFN_MASK_GATHER_LOAD:
case IFN_MASK_SCATTER_STORE:
+ return 4;
+
case IFN_MASK_LEN_GATHER_LOAD:
case IFN_MASK_LEN_SCATTER_STORE:
- return 4;
+ /* In tree-vect-patterns.cc, we will have these 2 situations:
+
+ - Unconditional gather load transforms
+ into MASK_LEN_GATHER_LOAD with no mask.
+
+ - Conditional gather load transforms
+ into MASK_LEN_GATHER_LOAD with real conditional mask.*/
+ if (!call || gimple_num_args (call) == 5)
+ return 4;
+ else
+ return -1;
default:
return (conditional_internal_fn_code (fn) != ERROR_MARK
@@ -235,7 +235,7 @@ extern bool can_interpret_as_conditional_op_p (gimple *, tree *,
extern bool internal_load_fn_p (internal_fn);
extern bool internal_store_fn_p (internal_fn);
extern bool internal_gather_scatter_fn_p (internal_fn);
-extern int internal_fn_mask_index (internal_fn);
+extern int internal_fn_mask_index (internal_fn, gcall * = nullptr);
extern int internal_fn_len_index (internal_fn);
extern int internal_fn_stored_value_index (internal_fn);
extern bool internal_gather_scatter_fn_supported_p (internal_fn, tree,
new file mode 100644
@@ -0,0 +1,15 @@
+/* { dg-do compile } */
+
+void
+f (int *restrict y, int *restrict x, int *restrict indices, int *restrict cond, int n)
+{
+ for (int i = 0; i < n; ++i)
+ {
+ if (cond[i * 2])
+ y[i * 2] = x[indices[i * 2]] + 1;
+ if (cond[i * 2 + 1])
+ y[i * 2 + 1] = x[indices[i * 2 + 1]] + 2;
+ }
+}
+
+/* { dg-final { scan-tree-dump "Loop contains only SLP stmts" vect { target vect_gather_load_ifn } } } */
@@ -6142,9 +6142,7 @@ vect_recog_gather_scatter_pattern (vec_info *vinfo,
mask = vect_convert_mask_for_vectype (mask, gs_vectype, stmt_info,
loop_vinfo);
else if (gs_info.ifn == IFN_MASK_SCATTER_STORE
- || gs_info.ifn == IFN_MASK_GATHER_LOAD
- || gs_info.ifn == IFN_MASK_LEN_SCATTER_STORE
- || gs_info.ifn == IFN_MASK_LEN_GATHER_LOAD)
+ || gs_info.ifn == IFN_MASK_GATHER_LOAD)
mask = build_int_cst (TREE_TYPE (truth_type_for (gs_vectype)), -1);
/* Get the invariant base and non-invariant offset, converting the
@@ -544,6 +544,16 @@ vect_get_operand_map (const gimple *stmt, unsigned char swap = 0)
case IFN_MASK_GATHER_LOAD:
return arg1_arg4_map;
+ case IFN_MASK_LEN_GATHER_LOAD:
+ /* In tree-vect-patterns.cc, we will have these 2 situations:
+
+ - Unconditional gather load transforms
+ into MASK_LEN_GATHER_LOAD with dummy mask which is -1.
+
+ - Conditional gather load transforms
+ into MASK_LEN_GATHER_LOAD with real conditional mask.*/
+ return gimple_num_args (call) == 5 ? arg1_arg4_map : arg1_map;
+
case IFN_MASK_STORE:
return arg3_arg2_map;
@@ -1077,7 +1087,8 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
if (cfn == CFN_MASK_LOAD
|| cfn == CFN_GATHER_LOAD
- || cfn == CFN_MASK_GATHER_LOAD)
+ || cfn == CFN_MASK_GATHER_LOAD
+ || cfn == CFN_MASK_LEN_GATHER_LOAD)
ldst_p = true;
else if (cfn == CFN_MASK_STORE)
{
@@ -1337,6 +1348,7 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
&& rhs_code != CFN_GATHER_LOAD
&& rhs_code != CFN_MASK_GATHER_LOAD
+ && rhs_code != CFN_MASK_LEN_GATHER_LOAD
/* Not grouped loads are handled as externals for BB
vectorization. For loop vectorization we can handle
splats the same we handle single element interleaving. */
@@ -1837,7 +1849,8 @@ vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD)
|| gimple_call_internal_p (stmt, IFN_GATHER_LOAD)
- || gimple_call_internal_p (stmt, IFN_MASK_GATHER_LOAD));
+ || gimple_call_internal_p (stmt, IFN_MASK_GATHER_LOAD)
+ || gimple_call_internal_p (stmt, IFN_MASK_LEN_GATHER_LOAD));
else
{
*max_nunits = this_max_nunits;
@@ -448,7 +448,7 @@ exist_non_indexing_operands_for_use_p (tree use, stmt_vec_info stmt_info)
if (call && gimple_call_internal_p (call))
{
internal_fn ifn = gimple_call_internal_fn (call);
- int mask_index = internal_fn_mask_index (ifn);
+ int mask_index = internal_fn_mask_index (ifn, call);
if (mask_index >= 0
&& use == gimple_call_arg (call, mask_index))
return true;
@@ -8246,7 +8246,7 @@ vectorizable_store (vec_info *vinfo,
if (!internal_store_fn_p (ifn))
return false;
- int mask_index = internal_fn_mask_index (ifn);
+ int mask_index = internal_fn_mask_index (ifn, call);
if (mask_index >= 0 && slp_node)
mask_index = vect_slp_child_index_for_operand (call, mask_index);
if (mask_index >= 0
@@ -9574,7 +9574,7 @@ vectorizable_load (vec_info *vinfo,
if (!scalar_dest)
return false;
- mask_index = internal_fn_mask_index (ifn);
+ mask_index = internal_fn_mask_index (ifn, call);
if (mask_index >= 0 && slp_node)
mask_index = vect_slp_child_index_for_operand (call, mask_index);
if (mask_index >= 0