@@ -291,6 +291,7 @@ public:
bool single_step_npatterns_p () const;
bool npatterns_all_equal_p () const;
+ rtx compact_mask () const;
machine_mode new_mode () const { return m_new_mode; }
scalar_mode inner_mode () const { return m_inner_mode; }
@@ -505,6 +506,44 @@ rvv_builder::npatterns_all_equal_p () const
return true;
}
+/* Generate the compact mask.
+
+ E.g: mask = { 0, -1 }, mode = VNx2BI, bitsize = 128bits.
+
+ GCC by default will generate the mask = 0b00000001xxxxx.
+
+ However, it's not expected mask for RVV since RVV
+ prefers the compact mask = 0b10xxxxx.
+*/
+rtx
+rvv_builder::compact_mask () const
+{
+ /* Use the container mode with SEW = 8 and LMUL = 1. */
+ unsigned container_size
+ = MAX (CEIL (npatterns (), 8), BYTES_PER_RISCV_VECTOR.to_constant () / 8);
+ machine_mode container_mode
+ = get_vector_mode (QImode, container_size).require ();
+
+ unsigned nunits = GET_MODE_NUNITS (container_mode).to_constant ();
+ rtvec v = rtvec_alloc (nunits);
+ for (unsigned i = 0; i < nunits; i++)
+ RTVEC_ELT (v, i) = const0_rtx;
+
+ unsigned char b = 0;
+ for (unsigned i = 0; i < npatterns (); i++)
+ {
+ if (INTVAL (elt (i)))
+ b = b | (1 << (i % 8));
+
+ if ((i > 0 && (i % 8) == 7) || (i == (npatterns () - 1)))
+ {
+ RTVEC_ELT (v, ((i + 7) / 8) - 1) = gen_int_mode (b, QImode);
+ b = 0;
+ }
+ }
+ return gen_rtx_CONST_VECTOR (container_mode, v);
+}
+
static unsigned
get_sew (machine_mode mode)
{
@@ -1141,11 +1180,23 @@ expand_const_vector (rtx target, rtx src)
if (GET_MODE_CLASS (mode) == MODE_VECTOR_BOOL)
{
rtx elt;
- gcc_assert (
- const_vec_duplicate_p (src, &elt)
- && (rtx_equal_p (elt, const0_rtx) || rtx_equal_p (elt, const1_rtx)));
- rtx ops[] = {target, src};
- emit_vlmax_insn (code_for_pred_mov (mode), RVV_UNOP, ops);
+ unsigned int nelts;
+ if (const_vec_duplicate_p (src, &elt))
+ {
+ rtx ops[] = {target, src};
+ emit_vlmax_insn (code_for_pred_mov (mode), RVV_UNOP, ops);
+ }
+ else if (GET_MODE_NUNITS (mode).is_constant (&nelts))
+ {
+ rvv_builder builder (mode, nelts, 1);
+ for (unsigned int i = 0; i < nelts; i++)
+ builder.quick_push (CONST_VECTOR_ELT (src, i));
+ rtx mask = builder.compact_mask ();
+ rtx mem = validize_mem (force_const_mem (GET_MODE (mask), mask));
+ emit_move_insn (target, gen_rtx_MEM (mode, XEXP (mem, 0)));
+ }
+ else
+ gcc_unreachable ();
return;
}
@@ -1323,6 +1323,12 @@ riscv_const_insns (rtx x)
return 1 + 4; /*vmv.v.x + memory access. */
}
}
+
+ /* GCC doesn't known RVV is using compact model of mask,
+ we should by default handle mask in mov<mode> pattern. */
+ if (GET_MODE_CLASS (GET_MODE (x)) == MODE_VECTOR_BOOL)
+ /* TODO: We can adjust it according real cost model of vlm.v. */
+ return 1;
}
/* TODO: We may support more const vector in the future. */
new file mode 100644
@@ -0,0 +1,23 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+#define N 16
+
+int
+main ()
+{
+ int mask[N] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ int64_t out[N] = {0};
+ for (int i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (int i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}
new file mode 100644
@@ -0,0 +1,23 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+#define N 16
+
+int
+main ()
+{
+ int mask[N] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ int out[N] = {0};
+ for (int i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (int i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}
new file mode 100644
@@ -0,0 +1,23 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+#define N 16
+
+int
+main ()
+{
+ int16_t mask[N] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ int16_t out[N] = {0};
+ for (int16_t i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (int16_t i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}
new file mode 100644
@@ -0,0 +1,23 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+#define N 16
+
+int
+main ()
+{
+ int8_t mask[N] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ int8_t out[N] = {0};
+ for (int8_t i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (int8_t i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}
new file mode 100644
@@ -0,0 +1,25 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax --param riscv-autovec-lmul=m2 -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+
+#define N 32
+
+int
+main ()
+{
+ int8_t mask[N] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ int8_t out[N] = {0};
+ for (int8_t i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (int8_t i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}
new file mode 100644
@@ -0,0 +1,27 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax --param riscv-autovec-lmul=m4 -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+
+#define N 64
+
+int
+main ()
+{
+ int8_t mask[N] = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ int8_t out[N] = {0};
+ for (int8_t i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (int8_t i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}
new file mode 100644
@@ -0,0 +1,30 @@
+/* { dg-do run { target { riscv_vector } } } */
+/* { dg-options "--param riscv-autovec-preference=fixed-vlmax --param riscv-autovec-lmul=m8 -O3" } */
+
+#include <stdint-gcc.h>
+#include <assert.h>
+
+#define N 128
+
+int
+main ()
+{
+ uint8_t mask[N]
+ = {0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1,
+ 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1};
+ uint8_t out[N] = {0};
+ for (uint8_t i = 0; i < N; ++i)
+ if (mask[i])
+ out[i] = i;
+ for (uint8_t i = 0; i < N; ++i)
+ {
+ if (mask[i])
+ assert (out[i] == i);
+ else
+ assert (out[i] == 0);
+ }
+}