@@ -91,6 +91,7 @@ (define_c_enum "unspec"
UNSPEC_MMA_XVI8GER4SPP
UNSPEC_MMA_XXMFACC
UNSPEC_MMA_XXMTACC
+ UNSPEC_MMA_VECTOR_PAIR_MEMORY
])
(define_c_enum "unspecv"
@@ -298,6 +299,49 @@ (define_insn_and_split "*movoo"
"TARGET_MMA
&& (gpc_reg_operand (operands[0], OOmode)
|| gpc_reg_operand (operands[1], OOmode))"
+{
+ if (MEM_P (operands[0]))
+ return TARGET_STORE_VECTOR_PAIR ? "stxvp%X0 %x1,%0" : "#";
+
+ if (MEM_P (operands[1]))
+ return TARGET_LOAD_VECTOR_PAIR ? "lxvp%X1 %x0,%1" : "#";
+
+ return "#";
+}
+ "&& reload_completed
+ && ((MEM_P (operands[0]) && !TARGET_STORE_VECTOR_PAIR)
+ || (MEM_P (operands[1]) && !TARGET_LOAD_VECTOR_PAIR)
+ || (!MEM_P (operands[0]) && !MEM_P (operands[1])))"
+ [(const_int 0)]
+{
+ rs6000_split_multireg_move (operands[0], operands[1]);
+ DONE;
+}
+ [(set_attr "type" "vecload,vecstore,veclogical")
+ (set_attr "size" "256")
+ (set_attr "length" "*,*,8")])
+
+;; Normally __builtin_vsx_lxvp and __builtin_vsx_stxvp are converted to a
+;; direct move insns, but if -mno-load-vector-pair or -mno-store-vector-pair
+;; are used, we use these insns to guarantee that the load vector pair is
+;; generated when the user explicitly uses the built-in function.
+(define_expand "lxvp_internal"
+ [(set (match_operand:OO 0 "gpc_reg_operand")
+ (unspec:OO [(mem:OO (match_operand 1 "address_operand"))]
+ UNSPEC_MMA_VECTOR_PAIR_MEMORY))]
+ "TARGET_MMA")
+
+(define_expand "stxvp_internal"
+ [(set (mem:OO (match_operand 0 "address_operand"))
+ (unspec:OO [(match_operand:OO 1 "gpc_reg_operand")]
+ UNSPEC_MMA_VECTOR_PAIR_MEMORY))]
+ "TARGET_MMA")
+
+(define_insn_and_split "*vector_pair_memory_builtin"
+ [(set (match_operand:OO 0 "nonimmediate_operand" "=wa,ZwO,wa")
+ (unspec:OO [(match_operand:OO 1 "input_operand" "ZwO,wa,wa")]
+ UNSPEC_MMA_VECTOR_PAIR_MEMORY))]
+ "TARGET_MMA"
"@
lxvp%X1 %x0,%1
stxvp%X0 %x1,%0
@@ -1,4 +1,4 @@
-/* Target-specific built-in function support for the Power architecture.
+ /* Target-specific built-in function support for the Power architecture.
See also rs6000-c.c, rs6000-gen-builtins.c, rs6000-builtins.def, and
rs6000-overloads.def.
Note that "normal" builtins (generic math functions, etc.) are handled
@@ -1165,9 +1165,23 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi,
if (TREE_TYPE (TREE_TYPE (ptr)) != vector_pair_type_node)
ptr = build1 (NOP_EXPR,
build_pointer_type (vector_pair_type_node), ptr);
- tree mem = build_simple_mem_ref (build2 (POINTER_PLUS_EXPR,
- TREE_TYPE (ptr), ptr, offset));
- gimplify_assign (lhs, mem, &new_seq);
+ tree addr = build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr, offset);
+
+ if (TARGET_LOAD_VECTOR_PAIR)
+ gimplify_assign (lhs, build_simple_mem_ref (addr), &new_seq);
+ else
+ {
+ tree ptrtype = build_pointer_type (vector_pair_type_node);
+ tree addrssa = create_tmp_reg_or_ssa_name (ptrtype);
+ tree lhsssa = create_tmp_reg_or_ssa_name (vector_pair_type_node);
+ gimplify_assign (addrssa, addr, &new_seq);
+ new_decl = rs6000_builtin_decls[RS6000_BIF_LXVP_INTERNAL];
+ new_call = gimple_build_call (new_decl, 1, addrssa);
+ gimple_call_set_lhs (new_call, lhsssa);
+ gimple_seq_add_stmt (&new_seq, new_call);
+ gimplify_assign (lhs, lhsssa, &new_seq);
+ }
+
pop_gimplify_context (NULL);
gsi_replace_with_seq (gsi, new_seq, true);
return true;
@@ -1182,9 +1196,20 @@ rs6000_gimple_fold_mma_builtin (gimple_stmt_iterator *gsi,
if (TREE_TYPE (TREE_TYPE (ptr)) != vector_pair_type_node)
ptr = build1 (NOP_EXPR,
build_pointer_type (vector_pair_type_node), ptr);
- tree mem = build_simple_mem_ref (build2 (POINTER_PLUS_EXPR,
- TREE_TYPE (ptr), ptr, offset));
- gimplify_assign (mem, src, &new_seq);
+ tree addr = build2 (POINTER_PLUS_EXPR, TREE_TYPE (ptr), ptr, offset);
+
+ if (TARGET_STORE_VECTOR_PAIR)
+ gimplify_assign (build_simple_mem_ref (addr), src, &new_seq);
+ else
+ {
+ tree ptrtype = build_pointer_type (vector_pair_type_node);
+ tree addrssa = create_tmp_reg_or_ssa_name (ptrtype);
+ gimplify_assign (addrssa, ptr, &new_seq);
+ new_decl = rs6000_builtin_decls[RS6000_BIF_STXVP_INTERNAL];
+ new_call = gimple_build_call (new_decl, 2, addrssa, src);
+ gimple_seq_add_stmt (&new_seq, new_call);
+ }
+
pop_gimplify_context (NULL);
gsi_replace_with_seq (gsi, new_seq, true);
return true;
@@ -3002,8 +3027,13 @@ mma_expand_builtin (tree exp, rtx target, insn_code icode,
rtx opnd;
const struct insn_operand_data *insn_op;
insn_op = &insn_data[icode].operand[nopnds];
+ /* The internal built-in functions for lxvp and stxvp must use normal
+ expansion to allow passing the address of a variable to the
+ built-in. */
if (TREE_CODE (arg) == ADDR_EXPR
- && MEM_P (DECL_RTL (TREE_OPERAND (arg, 0))))
+ && MEM_P (DECL_RTL (TREE_OPERAND (arg, 0)))
+ && fcode != RS6000_BIF_LXVP_INTERNAL
+ && fcode != RS6000_BIF_STXVP_INTERNAL)
opnd = DECL_RTL (TREE_OPERAND (arg, 0));
else
opnd = expand_normal (arg);
@@ -4129,5 +4129,11 @@
v256 __builtin_vsx_lxvp (unsigned long, const v256 *);
LXVP nothing {mma}
+ v256 __builtin_vsx_lxvp_internal (const v256 *);
+ LXVP_INTERNAL lxvp_internal {mma}
+
void __builtin_vsx_stxvp (v256, unsigned long, const v256 *);
STXVP nothing {mma,pair}
+
+ void __builtin_vsx_stxvp_internal (v256 *, v256);
+ STXVP_INTERNAL stxvp_internal {mma}
@@ -77,10 +77,12 @@
/* Flags that need to be turned off if -mno-power10. */
/* We comment out PCREL_OPT here to disable it by default because SPEC2017
performance was degraded by it. */
-#define OTHER_POWER10_MASKS (OPTION_MASK_MMA \
+#define OTHER_POWER10_MASKS (OPTION_MASK_LOAD_VECTOR_PAIR \
+ | OPTION_MASK_MMA \
| OPTION_MASK_PCREL \
/* | OPTION_MASK_PCREL_OPT */ \
- | OPTION_MASK_PREFIXED)
+ | OPTION_MASK_PREFIXED \
+ | OPTION_MASK_STORE_VECTOR_PAIR)
#define ISA_3_1_MASKS_SERVER (ISA_3_0_MASKS_SERVER \
| OPTION_MASK_POWER10 \
@@ -134,6 +136,7 @@
| OPTION_MASK_P10_FUSION \
| OPTION_MASK_HTM \
| OPTION_MASK_ISEL \
+ | OPTION_MASK_LOAD_VECTOR_PAIR \
| OPTION_MASK_MFCRF \
| OPTION_MASK_MMA \
| OPTION_MASK_MODULO \
@@ -156,6 +159,7 @@
| OPTION_MASK_QUAD_MEMORY_ATOMIC \
| OPTION_MASK_RECIP_PRECISION \
| OPTION_MASK_SOFT_FLOAT \
+ | OPTION_MASK_STORE_VECTOR_PAIR \
| OPTION_MASK_STRICT_ALIGN_OPTIONAL \
| OPTION_MASK_VSX)
@@ -2711,7 +2711,9 @@ rs6000_setup_reg_addr_masks (void)
/* Vector pairs can do both indexed and offset loads if the
instructions are enabled, otherwise they can only do offset loads
since it will be broken into two vector moves. Vector quads can
- only do offset loads. */
+ only do offset loads. If the user restricted generation of either
+ of the LXVP or STXVP instructions, do not allow indexed mode so
+ that we can split the load/store. */
else if ((addr_mask != 0) && TARGET_MMA
&& (m2 == OOmode || m2 == XOmode))
{
@@ -2719,7 +2721,9 @@ rs6000_setup_reg_addr_masks (void)
if (rc == RELOAD_REG_FPR || rc == RELOAD_REG_VMX)
{
addr_mask |= RELOAD_REG_QUAD_OFFSET;
- if (m2 == OOmode)
+ if (m2 == OOmode
+ && TARGET_LOAD_VECTOR_PAIR
+ && TARGET_STORE_VECTOR_PAIR)
addr_mask |= RELOAD_REG_INDEXED;
}
}
@@ -4405,6 +4409,26 @@ rs6000_option_override_internal (bool global_init_p)
rs6000_isa_flags &= ~OPTION_MASK_MMA;
}
+ /* Warn if -m-load-vector-pair or -m-store-vector-pair are used and MMA is
+ not set. */
+ if (!TARGET_MMA && TARGET_LOAD_VECTOR_PAIR)
+ {
+ if ((rs6000_isa_flags_explicit & OPTION_MASK_LOAD_VECTOR_PAIR) != 0)
+ warning (0, "%qs should not be used unless you use %qs",
+ "-mload-vector-pair", "-mmma");
+
+ rs6000_isa_flags &= ~OPTION_MASK_LOAD_VECTOR_PAIR;
+ }
+
+ if (!TARGET_MMA && TARGET_STORE_VECTOR_PAIR)
+ {
+ if ((rs6000_isa_flags_explicit & OPTION_MASK_STORE_VECTOR_PAIR) != 0)
+ warning (0, "%qs should not be used unless you use %qs",
+ "-mstore-vector-pair", "-mmma");
+
+ rs6000_isa_flags &= OPTION_MASK_STORE_VECTOR_PAIR;
+ }
+
/* Enable power10 fusion if we are tuning for power10, even if we aren't
generating power10 instructions. */
if (!(rs6000_isa_flags_explicit & OPTION_MASK_P10_FUSION))
@@ -24437,6 +24461,7 @@ static struct rs6000_opt_mask const rs6000_opt_masks[] =
{ "hard-dfp", OPTION_MASK_DFP, false, true },
{ "htm", OPTION_MASK_HTM, false, true },
{ "isel", OPTION_MASK_ISEL, false, true },
+ { "load-vector-pair", OPTION_MASK_LOAD_VECTOR_PAIR, false, true },
{ "mfcrf", OPTION_MASK_MFCRF, false, true },
{ "mfpgpr", 0, false, true },
{ "mma", OPTION_MASK_MMA, false, true },
@@ -24461,6 +24486,7 @@ static struct rs6000_opt_mask const rs6000_opt_masks[] =
{ "quad-memory-atomic", OPTION_MASK_QUAD_MEMORY_ATOMIC, false, true },
{ "recip-precision", OPTION_MASK_RECIP_PRECISION, false, true },
{ "save-toc-indirect", OPTION_MASK_SAVE_TOC_INDIRECT, false, true },
+ { "store-vector-pair", OPTION_MASK_STORE_VECTOR_PAIR, false, true },
{ "string", 0, false, true },
{ "update", OPTION_MASK_NO_UPDATE, true , true },
{ "vsx", OPTION_MASK_VSX, false, true },
@@ -597,6 +597,14 @@ mmma
Target Mask(MMA) Var(rs6000_isa_flags)
Generate (do not generate) MMA instructions.
+mload-vector-pair
+Target Undocumented Mask(LOAD_VECTOR_PAIR) Var(rs6000_isa_flags)
+Generate (do not generate) load vector pair instructions.
+
+mstore-vector-pair
+Target Undocumented Mask(STORE_VECTOR_PAIR) Var(rs6000_isa_flags)
+Generate (do not generate) store vector pair instructions.
+
mrelative-jumptables
Target Undocumented Var(rs6000_relative_jumptables) Init(1) Save
new file mode 100644
@@ -0,0 +1,39 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2" } */
+
+/* Test if we can control generating load and store vector pair via the target
+ attribute. */
+
+__attribute__((__target__("load-vector-pair,store-vector-pair")))
+void
+test_load_store (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 1 lxvp, 1 stxvp. */
+}
+
+__attribute__((__target__("load-vector-pair,no-store-vector-pair")))
+void
+test_load_no_store (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 1 lxvp, 2 stxv. */
+}
+
+__attribute__((__target__("no-load-vector-pair,store-vector-pair")))
+void
+test_store_no_load (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 1 stxvp. */
+}
+
+__attribute__((__target__("no-load-vector-pair,no-store-vector-pair")))
+void
+test_no_load_or_store (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 2 stxv. */
+}
+
+/* { dg-final { scan-assembler-times {\mp?lxvpx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mp?stxvpx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mp?lxvx?\M} 4 } } */
+/* { dg-final { scan-assembler-times {\mp?stxvx?\M} 4 } } */
new file mode 100644
@@ -0,0 +1,40 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2 -mno-load-vector-pair -mno-store-vector-pair" } */
+
+/* Test if we do not generate load and store vector pair if directed to on
+ power 10 for normal loads and stores, but the built-in versions still
+ generate the load/store vector pair instructions. Also check that the
+ prefixed plxvp or pstxvp are generated when appropriate. */
+
+static __vector_pair vp;
+
+void foo_assign (__vector_pair *p, const __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 2 stxv. */
+}
+
+void foo_builtin (__vector_pair *p, const __vector_pair *q)
+{
+ /* 1 lxvp, 1 stxvp. */
+ __builtin_vsx_stxvp (__builtin_vsx_lxvp (16, q), 32, p);
+}
+
+void foo_builtin_static_load (__vector_pair *p)
+{
+ /* 1 plxvp, 1 stxvp. */
+ __builtin_vsx_stxvp (__builtin_vsx_lxvp (0, &vp), 0, p);
+}
+
+void foo_builtin_static_store (const __vector_pair *p)
+{
+ /* 1 lxvp, 1 stxvp. */
+ __builtin_vsx_stxvp (__builtin_vsx_lxvp (0, p), 0, &vp);
+}
+
+/* { dg-final { scan-assembler-times {\mlxvx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mlxvpx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mplxvp\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mpstxvp\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mstxvx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mstxvpx?\M} 2 } } */
new file mode 100644
@@ -0,0 +1,55 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2" } */
+
+/* Test if we can control generating load and store vector pair via the #pragma
+ directive. */
+
+#pragma gcc push_options
+#pragma GCC target("load-vector-pair,store-vector-pair")
+
+void
+test_load_store (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 1 lxvp, 1 stxvp. */
+}
+
+#pragma gcc pop_options
+
+#pragma gcc push_options
+#pragma GCC target("load-vector-pair,no-store-vector-pair")
+
+void
+test_load_no_store (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 1 lxvp, 2 stxv. */
+}
+
+#pragma gcc pop_options
+
+#pragma gcc push_options
+#pragma GCC target("no-load-vector-pair,store-vector-pair")
+
+void
+test_store_no_load (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 1 stxvp. */
+}
+
+#pragma gcc pop_options
+
+#pragma gcc push_options
+#pragma GCC target("no-load-vector-pair,no-store-vector-pair")
+
+void
+test_no_load_or_store (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 2 stxv. */
+}
+
+#pragma gcc pop_options
+
+/* { dg-final { scan-assembler-times {\mp?lxvpx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mp?stxvpx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mp?lxvx?\M} 4 } } */
+/* { dg-final { scan-assembler-times {\mp?stxvx?\M} 4 } } */
new file mode 100644
@@ -0,0 +1,16 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2" } */
+
+/* Test if we generate load and store vector pair by default on power 10. */
+
+void
+test (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 1 lxvp, 1 stxvp. */
+}
+
+/* { dg-final { scan-assembler-times {\mp?lxvpx?\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mp?stxvpx?\M} 1 } } */
+/* { dg-final { scan-assembler-not {\mp?lxvx?\M} } } */
+/* { dg-final { scan-assembler-not {\mp?stxvx?\M} } } */
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2 -mno-store-vector-pair" } */
+
+/* Test if we generate load vector pair but not store vector pair if
+ -mno-store-vector-pair is used on power10. */
+
+void
+test (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 1 lxvp, 2 stxv. */
+}
+
+/* { dg-final { scan-assembler-times {\mp?lxvpx?\M} 1 } } */
+/* { dg-final { scan-assembler-not {\mp?stxvpx?\M} } } */
+/* { dg-final { scan-assembler-not {\mp?lxvx?\M} } } */
+/* { dg-final { scan-assembler-times {\mp?stxvx?\M} 2 } } */
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2 -mno-load-vector-pair" } */
+
+/* Test if we do not generate load vector pair but generate store vector pair
+ if -mno-load-vector-pair is used on power10. */
+
+void
+test (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 1 stxvp. */
+}
+
+/* { dg-final { scan-assembler-not {\mp?lxvpx?\M} } } */
+/* { dg-final { scan-assembler-times {\mp?stxvpx?\M} 1 } } */
+/* { dg-final { scan-assembler-times {\mp?lxvx?\M} 2 } } */
+/* { dg-final { scan-assembler-not {\mp?stxvx?\M} } } */
new file mode 100644
@@ -0,0 +1,17 @@
+/* { dg-do compile } */
+/* { dg-require-effective-target power10_ok } */
+/* { dg-options "-mdejagnu-cpu=power10 -O2 -mno-load-vector-pair -mno-store-vector-pair" } */
+
+/* Test if we do not generate load and store vector pair if directed to on
+ power 10. */
+
+void
+test (__vector_pair *p, __vector_pair *q)
+{
+ *p = *q; /* 2 lxv, 2 stxv. */
+}
+
+/* { dg-final { scan-assembler-not {\mp?lxvpx?\M} } } */
+/* { dg-final { scan-assembler-not {\mp?stxvpx?\M} } } */
+/* { dg-final { scan-assembler-times {\mp?lxvx?\M} 2 } } */
+/* { dg-final { scan-assembler-times {\mp?stxvx?\M} 2 } } */