i386: Auto vectorize usdot_prod, udot_prod with AVXVNNIINT16 instruction.
Checks
Commit Message
Hi all,
This patch aims to auto vectorize usdot_prod and udot_prod with newly
introduced AVX-VNNI-INT16.
Also I refined the redundant mode iterator in the patch.
Regtested on x86_64-pc-linux-gnu. Ok for trunk after AVX-VNNI-INT16 patch
checked in?
BRs,
Haochen
gcc/ChangeLog:
* config/i386/sse.md (VI2_AVX2): Delete V32HI since we actually
have the same iterator. Also renaming all the occurence to
VI2_AVX2_AVX512BW.
(usdot_prod<mode>): New define_expand.
(udot_prod<mode>): Ditto.
gcc/testsuite/ChangeLog:
* gcc.target/i386/vnniint16-auto-vectorize-1.c: New test.
* gcc.target/i386/vnniint16-auto-vectorize-2.c: Ditto.
---
gcc/config/i386/sse.md | 98 +++++++++++++------
.../i386/vnniint16-auto-vectorize-1.c | 28 ++++++
.../i386/vnniint16-auto-vectorize-2.c | 76 ++++++++++++++
3 files changed, 172 insertions(+), 30 deletions(-)
create mode 100644 gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-2.c
Comments
On Fri, Jul 14, 2023 at 8:24 AM Haochen Jiang <haochen.jiang@intel.com> wrote:
>
> Hi all,
>
> This patch aims to auto vectorize usdot_prod and udot_prod with newly
> introduced AVX-VNNI-INT16.
>
> Also I refined the redundant mode iterator in the patch.
>
> Regtested on x86_64-pc-linux-gnu. Ok for trunk after AVX-VNNI-INT16 patch
> checked in?
>
> BRs,
> Haochen
>
> gcc/ChangeLog:
>
> * config/i386/sse.md (VI2_AVX2): Delete V32HI since we actually
> have the same iterator. Also renaming all the occurence to
> VI2_AVX2_AVX512BW.
> (usdot_prod<mode>): New define_expand.
> (udot_prod<mode>): Ditto.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/i386/vnniint16-auto-vectorize-1.c: New test.
> * gcc.target/i386/vnniint16-auto-vectorize-2.c: Ditto.
OK with two changes below.
Thanks,
Uros.
> ---
> gcc/config/i386/sse.md | 98 +++++++++++++------
> .../i386/vnniint16-auto-vectorize-1.c | 28 ++++++
> .../i386/vnniint16-auto-vectorize-2.c | 76 ++++++++++++++
> 3 files changed, 172 insertions(+), 30 deletions(-)
> create mode 100644 gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-1.c
> create mode 100644 gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-2.c
>
> diff --git a/gcc/config/i386/sse.md b/gcc/config/i386/sse.md
> index 7471932b27e..98e7f9334bc 100644
> --- a/gcc/config/i386/sse.md
> +++ b/gcc/config/i386/sse.md
> @@ -545,6 +545,9 @@
> V32HI (V16HI "TARGET_AVX512VL")])
>
> (define_mode_iterator VI2_AVX2
> + [(V16HI "TARGET_AVX2") V8HI])
> +
> +(define_mode_iterator VI2_AVX2_AVX512BW
> [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
>
> (define_mode_iterator VI2_AVX512F
> @@ -637,9 +640,6 @@
> (V16HI "TARGET_AVX2") V8HI
> (V8SI "TARGET_AVX2") V4SI])
>
> -(define_mode_iterator VI2_AVX2_AVX512BW
> - [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
> -
> (define_mode_iterator VI248_AVX512VL
> [V32HI V16SI V8DI
> (V16HI "TARGET_AVX512VL") (V8SI "TARGET_AVX512VL")
> @@ -15298,16 +15298,16 @@
> })
>
> (define_expand "mul<mode>3<mask_name>"
> - [(set (match_operand:VI2_AVX2 0 "register_operand")
> - (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "vector_operand")
> - (match_operand:VI2_AVX2 2 "vector_operand")))]
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand")
> + (mult:VI2_AVX2_AVX512BW (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand")
> + (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand")))]
> "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
> "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
>
> (define_insn "*mul<mode>3<mask_name>"
> - [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,<v_Yw>")
> - (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "vector_operand" "%0,<v_Yw>")
> - (match_operand:VI2_AVX2 2 "vector_operand" "xBm,<v_Yw>m")))]
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,<v_Yw>")
> + (mult:VI2_AVX2_AVX512BW (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand" "%0,<v_Yw>")
> + (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand" "xBm,<v_Yw>m")))]
> "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
> && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
> "@
> @@ -15320,28 +15320,28 @@
> (set_attr "mode" "<sseinsnmode>")])
>
> (define_expand "<s>mul<mode>3_highpart<mask_name>"
> - [(set (match_operand:VI2_AVX2 0 "register_operand")
> - (truncate:VI2_AVX2
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand")
> + (truncate:VI2_AVX2_AVX512BW
> (lshiftrt:<ssedoublemode>
> (mult:<ssedoublemode>
> (any_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 1 "vector_operand"))
> + (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand"))
> (any_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 2 "vector_operand")))
> + (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand")))
> (const_int 16))))]
> "TARGET_SSE2
> && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
> "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
>
> (define_insn "*<s>mul<mode>3_highpart<mask_name>"
> - [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,<v_Yw>")
> - (truncate:VI2_AVX2
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,<v_Yw>")
> + (truncate:VI2_AVX2_AVX512BW
> (lshiftrt:<ssedoublemode>
> (mult:<ssedoublemode>
> (any_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 1 "vector_operand" "%0,<v_Yw>"))
> + (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand" "%0,<v_Yw>"))
> (any_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 2 "vector_operand" "xBm,<v_Yw>m")))
> + (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand" "xBm,<v_Yw>m")))
> (const_int 16))))]
> "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
> && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
> @@ -15591,8 +15591,8 @@
> (define_insn "avx512bw_pmaddwd512<mode><mask_name>"
> [(set (match_operand:<sseunpackmode> 0 "register_operand" "=v")
> (unspec:<sseunpackmode>
> - [(match_operand:VI2_AVX2 1 "register_operand" "v")
> - (match_operand:VI2_AVX2 2 "nonimmediate_operand" "vm")]
> + [(match_operand:VI2_AVX2_AVX512BW 1 "register_operand" "v")
> + (match_operand:VI2_AVX2_AVX512BW 2 "nonimmediate_operand" "vm")]
> UNSPEC_PMADDWD512))]
> "TARGET_AVX512BW && <mask_mode512bit_condition>"
> "vpmaddwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}";
> @@ -21569,16 +21569,16 @@
> })
>
> (define_expand "smulhrs<mode>3"
> - [(set (match_operand:VI2_AVX2 0 "register_operand")
> - (truncate:VI2_AVX2
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand")
> + (truncate:VI2_AVX2_AVX512BW
> (lshiftrt:<ssedoublemode>
> (plus:<ssedoublemode>
> (lshiftrt:<ssedoublemode>
> (mult:<ssedoublemode>
> (sign_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 1 "nonimmediate_operand"))
> + (match_operand:VI2_AVX2_AVX512BW 1 "nonimmediate_operand"))
> (sign_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 2 "nonimmediate_operand")))
> + (match_operand:VI2_AVX2_AVX512BW 2 "nonimmediate_operand")))
> (const_int 14))
> (match_dup 3))
> (const_int 1))))]
> @@ -21589,18 +21589,18 @@
> })
>
> (define_insn "*<ssse3_avx2>_pmulhrsw<mode>3<mask_name>"
> - [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,<v_Yw>")
> - (truncate:VI2_AVX2
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,<v_Yw>")
> + (truncate:VI2_AVX2_AVX512BW
> (lshiftrt:<ssedoublemode>
> (plus:<ssedoublemode>
> (lshiftrt:<ssedoublemode>
> (mult:<ssedoublemode>
> (sign_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 1 "vector_operand" "%0,<v_Yw>"))
> + (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand" "%0,<v_Yw>"))
> (sign_extend:<ssedoublemode>
> - (match_operand:VI2_AVX2 2 "vector_operand" "xBm,<v_Yw>m")))
> + (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand" "xBm,<v_Yw>m")))
> (const_int 14))
> - (match_operand:VI2_AVX2 3 "const1_operand"))
> + (match_operand:VI2_AVX2_AVX512BW 3 "const1_operand"))
> (const_int 1))))]
> "TARGET_SSSE3 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
> && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
> @@ -22327,8 +22327,8 @@
> (set_attr "mode" "<sseinsnmode>")])
>
> (define_insn "<sse4_1_avx2>_packusdw<mask_name>"
> - [(set (match_operand:VI2_AVX2 0 "register_operand" "=Yr,*x,<v_Yw>")
> - (unspec:VI2_AVX2
> + [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=Yr,*x,<v_Yw>")
> + (unspec:VI2_AVX2_AVX512BW
> [(match_operand:<sseunpackmode> 1 "register_operand" "0,0,<v_Yw>")
> (match_operand:<sseunpackmode> 2 "vector_operand" "YrBm,*xBm,<v_Yw>m")]
> UNSPEC_US_TRUNCATE))]
> @@ -30340,6 +30340,44 @@
> (UNSPEC_VPDPWSUD "wsud") (UNSPEC_VPDPWSUDS "wsuds")
> (UNSPEC_VPDPWUUD "wuud") (UNSPEC_VPDPWUUDS "wuuds")])
>
> +(define_expand "usdot_prod<mode>"
> + [(match_operand:<sseunpackmode> 0 "register_operand")
> + (match_operand:VI2_AVX2 1 "register_operand")
> + (match_operand:VI2_AVX2 2 "register_operand")
> + (match_operand:<sseunpackmode> 3 "register_operand")]
> + "TARGET_AVXVNNIINT16"
> +{
> + operands[1] = lowpart_subreg (<sseunpackmode>mode,
> + force_reg (<MODE>mode, operands[1]),
> + <MODE>mode);
> + operands[2] = lowpart_subreg (<sseunpackmode>mode,
> + force_reg (<MODE>mode, operands[2]),
> + <MODE>mode);
> + emit_insn (gen_rtx_SET (operands[0], operands[3]));
You don't have to emit a move, the register allocator will do that for you.
> + emit_insn (gen_vpdpwusd_<SDOT_VPDP_SUF> (operands[0], operands[3],
> + operands[1], operands[2]));
> + DONE;
> +})
> +
> +(define_expand "udot_prod<mode>"
> + [(match_operand:<sseunpackmode> 0 "register_operand")
> + (match_operand:VI2_AVX2 1 "register_operand")
> + (match_operand:VI2_AVX2 2 "register_operand")
> + (match_operand:<sseunpackmode> 3 "register_operand")]
> + "TARGET_AVXVNNIINT16"
> +{
> + operands[1] = lowpart_subreg (<sseunpackmode>mode,
> + force_reg (<MODE>mode, operands[1]),
> + <MODE>mode);
> + operands[2] = lowpart_subreg (<sseunpackmode>mode,
> + force_reg (<MODE>mode, operands[2]),
> + <MODE>mode);
> + emit_insn (gen_rtx_SET (operands[0], operands[3]));
Also here, the above is not needed.
> + emit_insn (gen_vpdpwuud_<SDOT_VPDP_SUF> (operands[0], operands[3],
> + operands[1], operands[2]));
> + DONE;
> +})
> +
> (define_insn "vpdp<vpdpwprodtype>_<mode>"
> [(set (match_operand:VI4_AVX 0 "register_operand" "=x")
> (unspec:VI4_AVX
> diff --git a/gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-1.c b/gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-1.c
> new file mode 100644
> index 00000000000..73f0d3296aa
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-1.c
> @@ -0,0 +1,28 @@
> +/* { dg-do compile } */
> +/* { dg-options "-mavxvnniint16 -O2" } */
> +/* { dg-final { scan-assembler "vpdpwusd\t" } } */
> +/* { dg-final { scan-assembler "vpdpwuud\t" } } */
> +
> +int __attribute__((noinline, noclone, optimize("tree-vectorize")))
> +usdot_prod_hi (unsigned short * restrict a, short * restrict b,
> + int c, int n)
> +{
> + int i;
> + for (i = 0; i < n; i++)
> + {
> + c += ((int) a[i] * (int) b[i]);
> + }
> + return c;
> +}
> +
> +int __attribute__((noinline, noclone, optimize("tree-vectorize")))
> +udot_prod_hi (unsigned short * restrict a, unsigned short *restrict b,
> + int c, int n)
> +{
> + int i;
> + for (i = 0; i < n; i++)
> + {
> + c += ((int) a[i] * (int) b[i]);
> + }
> + return c;
> +}
> diff --git a/gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-2.c b/gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-2.c
> new file mode 100644
> index 00000000000..90dc0eade7e
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/vnniint16-auto-vectorize-2.c
> @@ -0,0 +1,76 @@
> +/* { dg-do run } */
> +/* { dg-options "-O2 -mavxvnniint16" } */
> +/* { dg-require-effective-target avxvnniint16 } */
> +
> +#define AVXVNNIINT16
> +#ifndef CHECK
> +#define CHECK "avx-check.h"
> +#endif
> +
> +#ifndef TEST
> +#define TEST avx_test
> +#endif
> +
> +#include CHECK
> +#include "vnniint16-auto-vectorize-1.c"
> +
> +#define N 256
> +
> +short a_i16[N];
> +unsigned short b_u16[N], c_u16[N], d_u16[N];
> +int i16_exp, i16_ref;
> +
> +int __attribute__((noinline, noclone, optimize("no-tree-vectorize")))
> +udot_prod_hi_scalar (unsigned short * restrict a, unsigned short * restrict b,
> + int c, int n)
> +{
> + int i;
> + for (i = 0; i < n; i++)
> + {
> + c += ((int) a[i] * (int) b[i]);
> + }
> + return c;
> +}
> +
> +int __attribute__((noinline, noclone, optimize("no-tree-vectorize")))
> +usdot_prod_hi_scalar (unsigned short * restrict a, short *restrict b,
> + int c, int n)
> +{
> + int i;
> + for (i = 0; i < n; i++)
> + {
> + c += ((int) a[i] * (int) b[i]);
> + }
> + return c;
> +}
> +
> +void init ()
> +{
> + int i;
> +
> + i16_exp = i16_ref = 65535;
> +
> + for (i = 0; i < N; i++)
> + {
> + a_i16[i] = -i + 2;
> + b_u16[i] = i * 2;
> + c_u16[i] = i * 3;
> + d_u16[i] = i * 4;
> + }
> +}
> +
> +void
> +TEST (void)
> +{
> + init ();
> + i16_exp = usdot_prod_hi (a_i16, b_u16, i16_exp, N);
> + i16_ref = usdot_prod_hi_scalar (a_i16, b_u16, i16_ref, N);
> + if (i16_exp != i16_ref)
> + abort ();
> +
> + init ();
> + i16_exp = udot_prod_hi (c_u16, d_u16, i16_exp, N);
> + i16_ref = udot_prod_hi_scalar (c_u16, d_u16, i16_ref, N);
> + if (i16_exp != i16_ref)
> + abort ();
> +}
> --
> 2.31.1
>
@@ -545,6 +545,9 @@
V32HI (V16HI "TARGET_AVX512VL")])
(define_mode_iterator VI2_AVX2
+ [(V16HI "TARGET_AVX2") V8HI])
+
+(define_mode_iterator VI2_AVX2_AVX512BW
[(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
(define_mode_iterator VI2_AVX512F
@@ -637,9 +640,6 @@
(V16HI "TARGET_AVX2") V8HI
(V8SI "TARGET_AVX2") V4SI])
-(define_mode_iterator VI2_AVX2_AVX512BW
- [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
-
(define_mode_iterator VI248_AVX512VL
[V32HI V16SI V8DI
(V16HI "TARGET_AVX512VL") (V8SI "TARGET_AVX512VL")
@@ -15298,16 +15298,16 @@
})
(define_expand "mul<mode>3<mask_name>"
- [(set (match_operand:VI2_AVX2 0 "register_operand")
- (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "vector_operand")
- (match_operand:VI2_AVX2 2 "vector_operand")))]
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand")
+ (mult:VI2_AVX2_AVX512BW (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand")
+ (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand")))]
"TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
"ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
(define_insn "*mul<mode>3<mask_name>"
- [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,<v_Yw>")
- (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "vector_operand" "%0,<v_Yw>")
- (match_operand:VI2_AVX2 2 "vector_operand" "xBm,<v_Yw>m")))]
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,<v_Yw>")
+ (mult:VI2_AVX2_AVX512BW (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand" "%0,<v_Yw>")
+ (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand" "xBm,<v_Yw>m")))]
"TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
&& <mask_mode512bit_condition> && <mask_avx512bw_condition>"
"@
@@ -15320,28 +15320,28 @@
(set_attr "mode" "<sseinsnmode>")])
(define_expand "<s>mul<mode>3_highpart<mask_name>"
- [(set (match_operand:VI2_AVX2 0 "register_operand")
- (truncate:VI2_AVX2
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand")
+ (truncate:VI2_AVX2_AVX512BW
(lshiftrt:<ssedoublemode>
(mult:<ssedoublemode>
(any_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 1 "vector_operand"))
+ (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand"))
(any_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 2 "vector_operand")))
+ (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand")))
(const_int 16))))]
"TARGET_SSE2
&& <mask_mode512bit_condition> && <mask_avx512bw_condition>"
"ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
(define_insn "*<s>mul<mode>3_highpart<mask_name>"
- [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,<v_Yw>")
- (truncate:VI2_AVX2
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,<v_Yw>")
+ (truncate:VI2_AVX2_AVX512BW
(lshiftrt:<ssedoublemode>
(mult:<ssedoublemode>
(any_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 1 "vector_operand" "%0,<v_Yw>"))
+ (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand" "%0,<v_Yw>"))
(any_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 2 "vector_operand" "xBm,<v_Yw>m")))
+ (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand" "xBm,<v_Yw>m")))
(const_int 16))))]
"TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
&& <mask_mode512bit_condition> && <mask_avx512bw_condition>"
@@ -15591,8 +15591,8 @@
(define_insn "avx512bw_pmaddwd512<mode><mask_name>"
[(set (match_operand:<sseunpackmode> 0 "register_operand" "=v")
(unspec:<sseunpackmode>
- [(match_operand:VI2_AVX2 1 "register_operand" "v")
- (match_operand:VI2_AVX2 2 "nonimmediate_operand" "vm")]
+ [(match_operand:VI2_AVX2_AVX512BW 1 "register_operand" "v")
+ (match_operand:VI2_AVX2_AVX512BW 2 "nonimmediate_operand" "vm")]
UNSPEC_PMADDWD512))]
"TARGET_AVX512BW && <mask_mode512bit_condition>"
"vpmaddwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}";
@@ -21569,16 +21569,16 @@
})
(define_expand "smulhrs<mode>3"
- [(set (match_operand:VI2_AVX2 0 "register_operand")
- (truncate:VI2_AVX2
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand")
+ (truncate:VI2_AVX2_AVX512BW
(lshiftrt:<ssedoublemode>
(plus:<ssedoublemode>
(lshiftrt:<ssedoublemode>
(mult:<ssedoublemode>
(sign_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 1 "nonimmediate_operand"))
+ (match_operand:VI2_AVX2_AVX512BW 1 "nonimmediate_operand"))
(sign_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 2 "nonimmediate_operand")))
+ (match_operand:VI2_AVX2_AVX512BW 2 "nonimmediate_operand")))
(const_int 14))
(match_dup 3))
(const_int 1))))]
@@ -21589,18 +21589,18 @@
})
(define_insn "*<ssse3_avx2>_pmulhrsw<mode>3<mask_name>"
- [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,<v_Yw>")
- (truncate:VI2_AVX2
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,<v_Yw>")
+ (truncate:VI2_AVX2_AVX512BW
(lshiftrt:<ssedoublemode>
(plus:<ssedoublemode>
(lshiftrt:<ssedoublemode>
(mult:<ssedoublemode>
(sign_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 1 "vector_operand" "%0,<v_Yw>"))
+ (match_operand:VI2_AVX2_AVX512BW 1 "vector_operand" "%0,<v_Yw>"))
(sign_extend:<ssedoublemode>
- (match_operand:VI2_AVX2 2 "vector_operand" "xBm,<v_Yw>m")))
+ (match_operand:VI2_AVX2_AVX512BW 2 "vector_operand" "xBm,<v_Yw>m")))
(const_int 14))
- (match_operand:VI2_AVX2 3 "const1_operand"))
+ (match_operand:VI2_AVX2_AVX512BW 3 "const1_operand"))
(const_int 1))))]
"TARGET_SSSE3 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
&& !(MEM_P (operands[1]) && MEM_P (operands[2]))"
@@ -22327,8 +22327,8 @@
(set_attr "mode" "<sseinsnmode>")])
(define_insn "<sse4_1_avx2>_packusdw<mask_name>"
- [(set (match_operand:VI2_AVX2 0 "register_operand" "=Yr,*x,<v_Yw>")
- (unspec:VI2_AVX2
+ [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=Yr,*x,<v_Yw>")
+ (unspec:VI2_AVX2_AVX512BW
[(match_operand:<sseunpackmode> 1 "register_operand" "0,0,<v_Yw>")
(match_operand:<sseunpackmode> 2 "vector_operand" "YrBm,*xBm,<v_Yw>m")]
UNSPEC_US_TRUNCATE))]
@@ -30340,6 +30340,44 @@
(UNSPEC_VPDPWSUD "wsud") (UNSPEC_VPDPWSUDS "wsuds")
(UNSPEC_VPDPWUUD "wuud") (UNSPEC_VPDPWUUDS "wuuds")])
+(define_expand "usdot_prod<mode>"
+ [(match_operand:<sseunpackmode> 0 "register_operand")
+ (match_operand:VI2_AVX2 1 "register_operand")
+ (match_operand:VI2_AVX2 2 "register_operand")
+ (match_operand:<sseunpackmode> 3 "register_operand")]
+ "TARGET_AVXVNNIINT16"
+{
+ operands[1] = lowpart_subreg (<sseunpackmode>mode,
+ force_reg (<MODE>mode, operands[1]),
+ <MODE>mode);
+ operands[2] = lowpart_subreg (<sseunpackmode>mode,
+ force_reg (<MODE>mode, operands[2]),
+ <MODE>mode);
+ emit_insn (gen_rtx_SET (operands[0], operands[3]));
+ emit_insn (gen_vpdpwusd_<SDOT_VPDP_SUF> (operands[0], operands[3],
+ operands[1], operands[2]));
+ DONE;
+})
+
+(define_expand "udot_prod<mode>"
+ [(match_operand:<sseunpackmode> 0 "register_operand")
+ (match_operand:VI2_AVX2 1 "register_operand")
+ (match_operand:VI2_AVX2 2 "register_operand")
+ (match_operand:<sseunpackmode> 3 "register_operand")]
+ "TARGET_AVXVNNIINT16"
+{
+ operands[1] = lowpart_subreg (<sseunpackmode>mode,
+ force_reg (<MODE>mode, operands[1]),
+ <MODE>mode);
+ operands[2] = lowpart_subreg (<sseunpackmode>mode,
+ force_reg (<MODE>mode, operands[2]),
+ <MODE>mode);
+ emit_insn (gen_rtx_SET (operands[0], operands[3]));
+ emit_insn (gen_vpdpwuud_<SDOT_VPDP_SUF> (operands[0], operands[3],
+ operands[1], operands[2]));
+ DONE;
+})
+
(define_insn "vpdp<vpdpwprodtype>_<mode>"
[(set (match_operand:VI4_AVX 0 "register_operand" "=x")
(unspec:VI4_AVX
new file mode 100644
@@ -0,0 +1,28 @@
+/* { dg-do compile } */
+/* { dg-options "-mavxvnniint16 -O2" } */
+/* { dg-final { scan-assembler "vpdpwusd\t" } } */
+/* { dg-final { scan-assembler "vpdpwuud\t" } } */
+
+int __attribute__((noinline, noclone, optimize("tree-vectorize")))
+usdot_prod_hi (unsigned short * restrict a, short * restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
+
+int __attribute__((noinline, noclone, optimize("tree-vectorize")))
+udot_prod_hi (unsigned short * restrict a, unsigned short *restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
new file mode 100644
@@ -0,0 +1,76 @@
+/* { dg-do run } */
+/* { dg-options "-O2 -mavxvnniint16" } */
+/* { dg-require-effective-target avxvnniint16 } */
+
+#define AVXVNNIINT16
+#ifndef CHECK
+#define CHECK "avx-check.h"
+#endif
+
+#ifndef TEST
+#define TEST avx_test
+#endif
+
+#include CHECK
+#include "vnniint16-auto-vectorize-1.c"
+
+#define N 256
+
+short a_i16[N];
+unsigned short b_u16[N], c_u16[N], d_u16[N];
+int i16_exp, i16_ref;
+
+int __attribute__((noinline, noclone, optimize("no-tree-vectorize")))
+udot_prod_hi_scalar (unsigned short * restrict a, unsigned short * restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
+
+int __attribute__((noinline, noclone, optimize("no-tree-vectorize")))
+usdot_prod_hi_scalar (unsigned short * restrict a, short *restrict b,
+ int c, int n)
+{
+ int i;
+ for (i = 0; i < n; i++)
+ {
+ c += ((int) a[i] * (int) b[i]);
+ }
+ return c;
+}
+
+void init ()
+{
+ int i;
+
+ i16_exp = i16_ref = 65535;
+
+ for (i = 0; i < N; i++)
+ {
+ a_i16[i] = -i + 2;
+ b_u16[i] = i * 2;
+ c_u16[i] = i * 3;
+ d_u16[i] = i * 4;
+ }
+}
+
+void
+TEST (void)
+{
+ init ();
+ i16_exp = usdot_prod_hi (a_i16, b_u16, i16_exp, N);
+ i16_ref = usdot_prod_hi_scalar (a_i16, b_u16, i16_ref, N);
+ if (i16_exp != i16_ref)
+ abort ();
+
+ init ();
+ i16_exp = udot_prod_hi (c_u16, d_u16, i16_exp, N);
+ i16_ref = udot_prod_hi_scalar (c_u16, d_u16, i16_ref, N);
+ if (i16_exp != i16_ref)
+ abort ();
+}