+class demand_systemAdd comment:/* Demand system is the RVV-based VSETVL info analysis tools wrapper.
It defines compatible rules for SEW/LMUL, POLICY and AVL.
Also, it provides 3 iterfaces avaiable_p, compatible_p and
merge for the VSETVL PASS analysis and optimization.
- avaiable_p: Determine whether the next info can get the
avaiable VSETVL status from previous info.
e.g. bb 2 (demand SEW = 32, LMUL = M2) -> bb 3 (demand RATIO = 16).
Since bb 2 demand info (SEW/LMUL = 32/2 = 16) satisfies the bb 3
demand, the VSETVL instruction in bb 3 can be elided.
avaiable_p (previous, next) is true in such situation.
- compatible_p: Determine whether prev_info is compatible with next_info
so that we can have a new merged info that is avaiable to both of them.
- merge: Merge the stricter demand information from
next_info into prev_info so that prev_info becomes available to next_info.
*/
+ inline bool eq_lmul_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_vlmul () == next.get_vlmul ();
+ }
+ inline bool eq_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ();
+ }
+ inline bool eq_sew_lmul_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return eq_lmul_p (prev, next) && eq_sew_p (prev, next);
+ }+ inline bool ge_next_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (next.get_ta () && prev.get_sew () > next.get_sew ());
+ }
+ inline bool ge_prev_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (prev.get_ta () && prev.get_sew () < next.get_sew ());
+ }
+ inline bool le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_sew () <= next.get_max_sew ();
+ }
+ inline bool le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_sew () <= prev.get_max_sew ();
+ }....Rename them into:xxx_equal_pxxx_ge_pxxx_le_p+ inline bool has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ratio () >= (next.get_sew () / 8);
+ }
+ inline bool has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_ratio () >= (prev.get_sew () / 8);
+ }has_prev_ratio_p -> prev_ratio_valid_for_next_sew_phas_next_ratio_p -> next_ratio_valid_for_prev_sew_p
+ inline bool ge_next_sew_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && eq_ratio_p (prev, next);
+ }-> sew_ge_and_ratio_eq_p+ inline bool comp_tail_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ta () || next.get_ta () || eq_tail_policy_p (prev, next);
+ }
+
+ inline bool comp_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ma () || next.get_ma () || eq_mask_policy_p (prev, next);
+ }
+
+ inline bool comp_tail_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return comp_tail_policy_p (prev, next) && comp_mask_policy_p (prev, next);
+ }
Remove and change
+DEF_POLICY_RULE (tail_mask_policy, tail_mask_policy, tail_mask_policy,
+ comp_tail_mask_policy_p, eq_tail_mask_policy_p,
+ use_tail_mask_policy)
+DEF_POLICY_RULE (tail_mask_policy, tail_policy_only, tail_mask_policy,
+ comp_tail_policy_p, eq_tail_policy_p, use_tail_policy)
+DEF_POLICY_RULE (tail_mask_policy, mask_policy_only, tail_mask_policy,
+ comp_mask_policy_p, eq_mask_policy_p, use_mask_policy)
+DEF_POLICY_RULE (tail_mask_policy, ignore_policy, tail_mask_policy, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (tail_policy_only, tail_mask_policy, tail_mask_policy,
+ comp_tail_policy_p, always_false, use_mask_policy)
+DEF_POLICY_RULE (tail_policy_only, tail_policy_only, tail_policy_only,
+ comp_tail_policy_p, eq_tail_policy_p, use_tail_policy)
+DEF_POLICY_RULE (tail_policy_only, mask_policy_only, tail_mask_policy,
+ always_true, always_false, use_mask_policy)
+DEF_POLICY_RULE (tail_policy_only, ignore_policy, tail_policy_only, always_true,
+ always_true, nop)
into xxx_eq_p.def_or_use_vl_p -> modify_or_use_vl_pdef_avl_p -> modify_avl_pdef_reg_between -> modify_reg_between_p
same_reg_avl_p -> reg_avl_equal_pequal_avl_p -> avl_equal_p
use_min_max_sew -> use_min_of_max_sew+ /* Can we move vsetvl info between prev_insn and next_insn safe? */
+ bool safe_move_avl_vl_p (insn_info *prev_insn, insn_info *next_insn,
+ const vsetvl_info &info, bool ignore_vl = false)
avl_vl_unmodified_between_p
Change comment: /* Return true if AVL/VL doesn't change in any path from prev_insn to next_insn. */
compatible_sew_lmul_with -> sew_lmul_compatible_p
merge_sew_lmul_with -> merge_sew_lmul
compatible_with -> compatible_pavailable_with -> available_pmerge_with -> merge
juzhe.zhong@rivai.ai
From: Lehua Ding
Date: 2023-10-17 19:34
To: gcc-patches
CC: juzhe.zhong; kito.cheng; rdapp.gcc; palmer; jeffreyalaw; lehua.ding
Subject: [PATCH V2 02/14] RISC-V: P2: Refactor and cleanup demand system
This sub-patch refactor the demand system. I split the demand information
into three parts. They are sew and lmul related (sew_lmul_demand_type),
tail and mask policy related (policy_demand_type) and avl related
(avl_demand_type). Then we define three interfaces avaiable_with,
compatible_with and merge_with. avaiable_with is used to determine whether
the two vsetvl infos prev_info and next_info are available or not. If
prev_info is available for next_info, it means that the RVV insn
corresponding to next_info on the path from prev_info to next_info
can be used without inserting a separate vsetvl instruction. compatible_with
is used to determine whether prev_info is compatible with next_info, and if
so, merge_with can be used to merge the stricter demand information from
next_info into prev_info so that prev_info becomes available to next_info.
gcc/ChangeLog:
* config/riscv/riscv-vsetvl.cc (incompatible_avl_p): Removed.
(different_sew_p): Removed.
(different_lmul_p): Removed.
(different_ratio_p): Removed.
(different_tail_policy_p): Removed.
(different_mask_policy_p): Removed.
(possible_zero_avl_p): Removed.
(second_ratio_invalid_for_first_sew_p): Removed.
(second_ratio_invalid_for_first_lmul_p): Removed.
(float_insn_valid_sew_p): Removed.
(second_sew_less_than_first_sew_p): Removed.
(first_sew_less_than_second_sew_p): Removed.
(compare_lmul): Removed.
(second_lmul_less_than_first_lmul_p): Removed.
(second_ratio_less_than_first_ratio_p): Removed.
(DEF_INCOMPATIBLE_COND): Removed.
(greatest_sew): Removed.
(first_sew): Removed.
(second_sew): Removed.
(first_vlmul): Removed.
(second_vlmul): Removed.
(first_ratio): Removed.
(second_ratio): Removed.
(vlmul_for_first_sew_second_ratio): Removed.
(vlmul_for_greatest_sew_second_ratio): Removed.
(ratio_for_second_sew_first_vlmul): Removed.
(DEF_SEW_LMUL_FUSE_RULE): Removed.
(always_unavailable): Removed.
(avl_unavailable_p): Removed.
(sew_unavailable_p): Removed.
(lmul_unavailable_p): Removed.
(ge_sew_unavailable_p): Removed.
(ge_sew_lmul_unavailable_p): Removed.
(ge_sew_ratio_unavailable_p): Removed.
(DEF_UNAVAILABLE_COND): Removed.
(same_sew_lmul_demand_p): Removed.
(propagate_avl_across_demands_p): Removed.
(reg_available_p): Removed.
(support_relaxed_compatible_p): Removed.
(class demand_system): New class.
(DEF_SEW_LMUL_RULE): New Marco.
(DEF_POLICY_RULE): New macro.
(DEF_AVL_RULE): New macro.
* config/riscv/riscv-vsetvl.def (DEF_INCOMPATIBLE_COND): Removed.
(DEF_SEW_LMUL_RULE): New macro.
(DEF_SEW_LMUL_FUSE_RULE): Removed.
(DEF_POLICY_RULE): New macro.
(DEF_UNAVAILABLE_COND): Removed.
(DEF_AVL_RULE): New macro.
(sew_lmul): New demand type.
(ratio_only): New demand type.
(sew_only): New demand type.
(ge_sew): New demand type.
(ratio_and_ge_sew): New demand type.
(tail_mask_policy): New demand type.
(tail_policy_only): New demand type.
(mask_policy_only): New demand type.
(ignore_policy): New demand type.
(avl): New demand type.
(non_zero_avl): New demand type.
(ignore_avl): New demand type.
* config/riscv/riscv-vsetvl.h (enum demand_type): Removed.
(enum demand_status): Removed.
(enum fusion_type): Removed.
(struct demands_pair): Removed.
(struct demands_cond): Removed.
(struct demands_fuse_rule): Removed.
---
gcc/config/riscv/riscv-vsetvl.cc | 1062 ++++++++++++++++++-----------
gcc/config/riscv/riscv-vsetvl.def | 634 ++++-------------
gcc/config/riscv/riscv-vsetvl.h | 79 ---
3 files changed, 814 insertions(+), 961 deletions(-)
diff --git a/gcc/config/riscv/riscv-vsetvl.cc b/gcc/config/riscv/riscv-vsetvl.cc
index 79ba8466556..be40b6fdf4c 100644
--- a/gcc/config/riscv/riscv-vsetvl.cc
+++ b/gcc/config/riscv/riscv-vsetvl.cc
@@ -1091,401 +1091,6 @@ calculate_vlmul (unsigned int sew, unsigned int ratio)
return LMUL_RESERVED;
}
-static bool
-incompatible_avl_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return !info1.compatible_avl_p (info2) && !info2.compatible_avl_p (info1);
-}
-
-static bool
-different_sew_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_sew () != info2.get_sew ();
-}
-
-static bool
-different_lmul_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_vlmul () != info2.get_vlmul ();
-}
-
-static bool
-different_ratio_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_ratio () != info2.get_ratio ();
-}
-
-static bool
-different_tail_policy_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_ta () != info2.get_ta ();
-}
-
-static bool
-different_mask_policy_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_ma () != info2.get_ma ();
-}
-
-static bool
-possible_zero_avl_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return !info1.has_non_zero_avl () || !info2.has_non_zero_avl ();
-}
-
-static bool
-second_ratio_invalid_for_first_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (info1.get_sew (), info2.get_ratio ())
- == LMUL_RESERVED;
-}
-
-static bool
-second_ratio_invalid_for_first_lmul_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_sew (info1.get_vlmul (), info2.get_ratio ()) == 0;
-}
-
-static bool
-float_insn_valid_sew_p (const vector_insn_info &info, unsigned int sew)
-{
- if (info.get_insn () && info.get_insn ()->is_real ()
- && get_attr_type (info.get_insn ()->rtl ()) == TYPE_VFMOVFV)
- {
- if (sew == 16)
- return TARGET_VECTOR_ELEN_FP_16;
- else if (sew == 32)
- return TARGET_VECTOR_ELEN_FP_32;
- else if (sew == 64)
- return TARGET_VECTOR_ELEN_FP_64;
- }
- return true;
-}
-
-static bool
-second_sew_less_than_first_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info2.get_sew () < info1.get_sew ()
- || !float_insn_valid_sew_p (info1, info2.get_sew ());
-}
-
-static bool
-first_sew_less_than_second_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_sew () < info2.get_sew ()
- || !float_insn_valid_sew_p (info2, info1.get_sew ());
-}
-
-/* return 0 if LMUL1 == LMUL2.
- return -1 if LMUL1 < LMUL2.
- return 1 if LMUL1 > LMUL2. */
-static int
-compare_lmul (vlmul_type vlmul1, vlmul_type vlmul2)
-{
- if (vlmul1 == vlmul2)
- return 0;
-
- switch (vlmul1)
- {
- case LMUL_1:
- if (vlmul2 == LMUL_2 || vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_2:
- if (vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_4:
- if (vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_8:
- return -1;
- case LMUL_F2:
- if (vlmul2 == LMUL_1 || vlmul2 == LMUL_2 || vlmul2 == LMUL_4
- || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_F4:
- if (vlmul2 == LMUL_F2 || vlmul2 == LMUL_1 || vlmul2 == LMUL_2
- || vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_F8:
- return 0;
- default:
- gcc_unreachable ();
- }
-}
-
-static bool
-second_lmul_less_than_first_lmul_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return compare_lmul (info2.get_vlmul (), info1.get_vlmul ()) == -1;
-}
-
-static bool
-second_ratio_less_than_first_ratio_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info2.get_ratio () < info1.get_ratio ();
-}
-
-static CONSTEXPR const demands_cond incompatible_conds[] = {
-#define DEF_INCOMPATIBLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, \
- GE_SEW1, TAIL_POLICTY1, MASK_POLICY1, AVL2, \
- SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, \
- TAIL_POLICTY2, MASK_POLICY2, COND) \
- {{{AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, TAIL_POLICTY1, \
- MASK_POLICY1}, \
- {AVL2, SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2}}, \
- COND},
-#include "riscv-vsetvl.def"
-};
-
-static unsigned
-greatest_sew (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return std::max (info1.get_sew (), info2.get_sew ());
-}
-
-static unsigned
-first_sew (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_sew ();
-}
-
-static unsigned
-second_sew (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_sew ();
-}
-
-static vlmul_type
-first_vlmul (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_vlmul ();
-}
-
-static vlmul_type
-second_vlmul (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_vlmul ();
-}
-
-static unsigned
-first_ratio (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_ratio ();
-}
-
-static unsigned
-second_ratio (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_ratio ();
-}
-
-static vlmul_type
-vlmul_for_first_sew_second_ratio (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (info1.get_sew (), info2.get_ratio ());
-}
-
-static vlmul_type
-vlmul_for_greatest_sew_second_ratio (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (MAX (info1.get_sew (), info2.get_sew ()),
- info2.get_ratio ());
-}
-
-static unsigned
-ratio_for_second_sew_first_vlmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_ratio (info2.get_sew (), info1.get_vlmul ());
-}
-
-static CONSTEXPR const demands_fuse_rule fuse_rules[] = {
-#define DEF_SEW_LMUL_FUSE_RULE(DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, \
- DEMAND_GE_SEW1, DEMAND_SEW2, DEMAND_LMUL2, \
- DEMAND_RATIO2, DEMAND_GE_SEW2, NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, NEW_SEW, NEW_VLMUL, \
- NEW_RATIO) \
- {{{DEMAND_ANY, DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, DEMAND_ANY, \
- DEMAND_GE_SEW1, DEMAND_ANY, DEMAND_ANY}, \
- {DEMAND_ANY, DEMAND_SEW2, DEMAND_LMUL2, DEMAND_RATIO2, DEMAND_ANY, \
- DEMAND_GE_SEW2, DEMAND_ANY, DEMAND_ANY}}, \
- NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, \
- NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, \
- NEW_SEW, \
- NEW_VLMUL, \
- NEW_RATIO},
-#include "riscv-vsetvl.def"
-};
-
-static bool
-always_unavailable (const vector_insn_info &, const vector_insn_info &)
-{
- return true;
-}
-
-static bool
-avl_unavailable_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return !info2.compatible_avl_p (info1.get_avl_info ());
-}
-
-static bool
-sew_unavailable_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL) && !info2.demand_p (DEMAND_RATIO))
- {
- if (info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return info1.get_sew () != info2.get_sew ();
- }
- return true;
-}
-
-static bool
-lmul_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.get_vlmul () == info2.get_vlmul () && !info2.demand_p (DEMAND_SEW)
- && !info2.demand_p (DEMAND_RATIO))
- return false;
- return true;
-}
-
-static bool
-ge_sew_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL) && !info2.demand_p (DEMAND_RATIO)
- && info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return true;
-}
-
-static bool
-ge_sew_lmul_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_RATIO) && info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return true;
-}
-
-static bool
-ge_sew_ratio_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL))
- {
- if (info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- /* Demand GE_SEW should be available for non-demand SEW. */
- else if (!info2.demand_p (DEMAND_SEW))
- return false;
- }
- return true;
-}
-
-static CONSTEXPR const demands_cond unavailable_conds[] = {
-#define DEF_UNAVAILABLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, \
- TAIL_POLICTY1, MASK_POLICY1, AVL2, SEW2, LMUL2, \
- RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2, COND) \
- {{{AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, TAIL_POLICTY1, \
- MASK_POLICY1}, \
- {AVL2, SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2}}, \
- COND},
-#include "riscv-vsetvl.def"
-};
-
-static bool
-same_sew_lmul_demand_p (const bool *dems1, const bool *dems2)
-{
- return dems1[DEMAND_SEW] == dems2[DEMAND_SEW]
- && dems1[DEMAND_LMUL] == dems2[DEMAND_LMUL]
- && dems1[DEMAND_RATIO] == dems2[DEMAND_RATIO] && !dems1[DEMAND_GE_SEW]
- && !dems2[DEMAND_GE_SEW];
-}
-
-static bool
-propagate_avl_across_demands_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info2.demand_p (DEMAND_AVL))
- {
- if (info2.demand_p (DEMAND_NONZERO_AVL))
- return info1.demand_p (DEMAND_AVL)
- && !info1.demand_p (DEMAND_NONZERO_AVL) && info1.has_avl_reg ();
- }
- else
- return info1.demand_p (DEMAND_AVL) && info1.has_avl_reg ();
- return false;
-}
-
-static bool
-reg_available_p (const insn_info *insn, const vector_insn_info &info)
-{
- if (info.has_avl_reg () && !info.get_avl_source ())
- return false;
- insn_info *def_insn = info.get_avl_source ()->insn ();
- if (def_insn->bb () == insn->bb ())
- return before_p (def_insn, insn);
- else
- return dominated_by_p (CDI_DOMINATORS, insn->bb ()->cfg_bb (),
- def_insn->bb ()->cfg_bb ());
-}
-
-/* Return true if the instruction support relaxed compatible check. */
-static bool
-support_relaxed_compatible_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (fault_first_load_p (info1.get_insn ()->rtl ())
- && info2.demand_p (DEMAND_AVL) && info2.has_avl_reg ()
- && info2.get_avl_source () && info2.get_avl_source ()->insn ()->is_phi ())
- {
- hash_set<set_info *> sets
- = get_all_sets (info2.get_avl_source (), true, false, false);
- for (set_info *set : sets)
- {
- if (read_vl_insn_p (set->insn ()->rtl ()))
- {
- const insn_info *insn
- = get_backward_fault_first_load_insn (set->insn ());
- if (insn == info1.get_insn ())
- return info2.compatible_vtype_p (info1);
- }
- }
- }
- return false;
-}
-
/* Count the number of REGNO in RINSN. */
static int
count_regno_occurrences (rtx_insn *rinsn, unsigned int regno)
@@ -2118,6 +1723,673 @@ public:
}
};
+class demand_system
+{
+private:
+ sbitmap *m_avl_def_in;
+ sbitmap *m_avl_def_out;
+
+ /* predictors. */
+
+ inline bool always_true (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {
+ return true;
+ }
+ inline bool always_false (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {
+ return false;
+ }
+
+ /* predictors for sew and lmul */
+
+ inline bool eq_lmul_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_vlmul () == next.get_vlmul ();
+ }
+ inline bool eq_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ();
+ }
+ inline bool eq_sew_lmul_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return eq_lmul_p (prev, next) && eq_sew_p (prev, next);
+ }
+ inline bool ge_next_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (next.get_ta () && prev.get_sew () > next.get_sew ());
+ }
+ inline bool ge_prev_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (prev.get_ta () && prev.get_sew () < next.get_sew ());
+ }
+ inline bool le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_sew () <= next.get_max_sew ();
+ }
+ inline bool le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_sew () <= prev.get_max_sew ();
+ }
+ inline bool max_sew_overlap_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return !(prev.get_sew () > next.get_max_sew ()
+ || next.get_sew () > prev.get_max_sew ());
+ }
+ inline bool eq_ratio_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.has_same_ratio (next);
+ }
+ inline bool has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ratio () >= (next.get_sew () / 8);
+ }
+ inline bool has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_ratio () >= (prev.get_sew () / 8);
+ }
+
+ inline bool ge_next_sew_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && eq_ratio_p (prev, next);
+ }
+ inline bool ge_next_sew_and_le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && le_next_max_sew_p (prev, next);
+ }
+ inline bool
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && le_next_max_sew_p (prev, next)
+ && has_next_ratio_p (prev, next);
+ }
+ inline bool ge_prev_sew_and_le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_prev_sew_p (prev, next) && le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return has_next_ratio_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+ inline bool
+ ge_prev_sew_and_le_prev_max_sew_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_prev_sew_p (prev, next) && eq_ratio_p (prev, next)
+ && le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return has_prev_ratio_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+ inline bool
+ ge_prev_sew_and_le_prev_max_sew_and_has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_prev_sew_p (prev, next) && has_prev_ratio_p (prev, next)
+ && le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return eq_ratio_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+
+ /* predictors for tail and mask policy */
+
+ inline bool eq_tail_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ta () == next.get_ta ();
+ }
+ inline bool eq_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ma () == next.get_ma ();
+ }
+ inline bool eq_tail_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return eq_tail_policy_p (prev, next) && eq_mask_policy_p (prev, next);
+ }
+
+ inline bool comp_tail_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ta () || next.get_ta () || eq_tail_policy_p (prev, next);
+ }
+
+ inline bool comp_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ma () || next.get_ma () || eq_mask_policy_p (prev, next);
+ }
+
+ inline bool comp_tail_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return comp_tail_policy_p (prev, next) && comp_mask_policy_p (prev, next);
+ }
+
+ /* predictors for avl */
+
+ inline bool def_or_use_vl_p (insn_info *i, const vsetvl_info &info)
+ {
+ return info.has_reg_vl ()
+ && (find_access (i->uses (), REGNO (info.get_vl ()))
+ || find_access (i->defs (), REGNO (info.get_vl ())));
+ }
+ inline bool def_avl_p (insn_info *i, const vsetvl_info &info)
+ {
+ return info.has_reg_avl ()
+ && find_access (i->defs (), REGNO (info.get_avl ()));
+ }
+
+ inline bool def_reg_between (insn_info *prev_insn, insn_info *curr_insn,
+ unsigned regno)
+ {
+ gcc_assert (prev_insn->compare_with (curr_insn) < 0);
+ for (insn_info *i = curr_insn->prev_nondebug_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def of regno
+ if (find_access (i->defs (), regno))
+ return true;
+ }
+ return false;
+ }
+
+ inline bool same_reg_avl_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!prev.has_reg_avl () || !next.has_reg_avl ())
+ return false;
+
+ if (same_equiv_note_p (prev.get_avl_def (), next.get_avl_def ()))
+ return true;
+
+ if (REGNO (prev.get_avl ()) != REGNO (next.get_avl ()))
+ return false;
+
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return safe_move_avl_vl_p (prev_insn, next_insn, next, false);
+ }
+
+ inline bool equal_avl_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+
+ if (prev.get_ratio () != next.get_ratio ())
+ return false;
+
+ if (next.has_reg_vl () && next.use_by_non_rvv_insn_p ())
+ return false;
+
+ if (vector_config_insn_p (prev.get_insn ()->rtl ()) && next.get_avl_def ()
+ && next.get_avl_def ()->insn () == prev.get_insn ())
+ return true;
+
+ if (prev.get_read_vl_insn ())
+ {
+ if (!next.has_reg_avl () || !next.get_avl_def ())
+ return false;
+ insn_info *avl_def_insn = extract_single_source (next.get_avl_def ());
+ return avl_def_insn == prev.get_read_vl_insn ();
+ }
+
+ if (prev == next && prev.has_reg_avl ())
+ {
+ insn_info *insn = prev.get_insn ();
+ bb_info *bb = insn->bb ();
+ for (insn_info *i = insn; real_insn_and_same_bb_p (i, bb);
+ i = i->next_nondebug_insn ())
+ if (find_access (i->defs (), REGNO (prev.get_avl ())))
+ return false;
+ }
+
+ if (prev.has_vlmax_avl () && next.has_vlmax_avl ())
+ return true;
+ else if (prev.has_imm_avl () && next.has_imm_avl ())
+ return INTVAL (prev.get_avl ()) == INTVAL (next.get_avl ());
+ else if (prev.has_reg_vl () && next.has_reg_avl ()
+ && REGNO (prev.get_vl ()) == REGNO (next.get_avl ()))
+ {
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return safe_move_avl_vl_p (prev_insn, next_insn, next, false);
+ }
+ else if (prev.has_reg_avl () && next.has_reg_avl ())
+ return same_reg_avl_p (prev, next);
+
+ return false;
+ }
+ inline bool equal_avl_or_prev_non_zero_avl_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return equal_avl_p (prev, next) || prev.has_non_zero_avl ();
+ }
+
+ inline bool can_use_next_avl_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ if (!next.has_reg_avl () && !next.has_reg_vl ())
+ return true;
+
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return safe_move_avl_vl_p (prev_insn, next_insn, next);
+ }
+
+ inline bool equal_avl_or_next_non_zero_avl_and_can_use_next_avl_p (
+ const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return equal_avl_p (prev, next)
+ || (next.has_non_zero_avl () && can_use_next_avl_p (prev, next));
+ }
+
+ /* modifiers */
+
+ inline void nop (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {}
+
+ /* modifiers for sew and lmul */
+
+ inline void use_min_max_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ prev.set_max_sew (MIN (prev.get_max_sew (), next.get_max_sew ()));
+ }
+ inline void use_next_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ prev.set_sew (next.get_sew ());
+ use_min_max_sew (prev, next);
+ }
+ inline void use_max_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ auto max_sew = std::max (prev.get_sew (), next.get_sew ());
+ prev.set_sew (max_sew);
+ use_min_max_sew (prev, next);
+ }
+ inline void use_next_sew_lmul (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ use_next_sew (prev, next);
+ prev.set_vlmul (next.get_vlmul ());
+ prev.set_ratio (next.get_ratio ());
+ }
+ inline void use_next_sew_with_prev_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ use_next_sew (prev, next);
+ prev.set_vlmul (calculate_vlmul (next.get_sew (), prev.get_ratio ()));
+ }
+ inline void modify_lmul_with_next_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ prev.set_vlmul (calculate_vlmul (prev.get_sew (), next.get_ratio ()));
+ prev.set_ratio (next.get_ratio ());
+ }
+
+ inline void use_max_sew_and_lmul_with_next_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ prev.set_vlmul (calculate_vlmul (prev.get_sew (), next.get_ratio ()));
+ use_max_sew (prev, next);
+ prev.set_ratio (next.get_ratio ());
+ }
+
+ inline void use_max_sew_and_lmul_with_prev_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ auto max_sew = std::max (prev.get_sew (), next.get_sew ());
+ prev.set_vlmul (calculate_vlmul (max_sew, prev.get_ratio ()));
+ prev.set_sew (max_sew);
+ }
+
+ /* modifiers for tail and mask policy */
+
+ inline void use_tail_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!next.get_ta ())
+ prev.set_ta (next.get_ta ());
+ }
+ inline void use_mask_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!next.get_ma ())
+ prev.set_ma (next.get_ma ());
+ }
+ inline void use_tail_mask_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ use_tail_policy (prev, next);
+ use_mask_policy (prev, next);
+ }
+
+ /* modifiers for avl */
+
+ inline void use_next_avl (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (can_use_next_avl_p (prev, next));
+ prev.update_avl (next);
+ }
+
+ inline void use_next_avl_when_not_equal (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ if (equal_avl_p (prev, next))
+ return;
+ gcc_assert (next.has_non_zero_avl ());
+ use_next_avl (prev, next);
+ }
+
+public:
+ demand_system () : m_avl_def_in (nullptr), m_avl_def_out (nullptr) {}
+
+ void set_avl_in_out_data (sbitmap *avl_def_in, sbitmap *avl_def_out)
+ {
+ m_avl_def_in = avl_def_in;
+ m_avl_def_out = avl_def_out;
+ }
+
+ /* Can we move vsetvl info between prev_insn and next_insn safe? */
+ bool safe_move_avl_vl_p (insn_info *prev_insn, insn_info *next_insn,
+ const vsetvl_info &info, bool ignore_vl = false)
+ {
+ gcc_assert ((ignore_vl && info.has_reg_avl ())
+ || (info.has_reg_avl () || info.has_reg_vl ()));
+
+ gcc_assert (!prev_insn->is_debug_insn () && !next_insn->is_debug_insn ());
+ if (prev_insn->bb () == next_insn->bb ()
+ && prev_insn->compare_with (next_insn) < 0)
+ {
+ for (insn_info *i = next_insn->prev_nondebug_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && def_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (def_avl_p (i, info))
+ return false;
+ }
+ return true;
+ }
+ else
+ {
+ if (!ignore_vl && info.has_reg_vl ())
+ {
+ bitmap live_out = df_get_live_out (prev_insn->bb ()->cfg_bb ());
+ if (bitmap_bit_p (live_out, REGNO (info.get_vl ())))
+ return false;
+ }
+
+ if (info.has_reg_avl () && m_avl_def_in && m_avl_def_out)
+ {
+ bool has_avl_out = false;
+ unsigned regno = REGNO (info.get_avl ());
+ unsigned expr_id;
+ sbitmap_iterator sbi;
+ EXECUTE_IF_SET_IN_BITMAP (m_avl_def_out[prev_insn->bb ()->index ()],
+ 0, expr_id, sbi)
+ {
+ if (get_regno (expr_id, last_basic_block_for_fn (cfun))
+ != regno)
+ continue;
+ has_avl_out = true;
+ if (!bitmap_bit_p (m_avl_def_in[next_insn->bb ()->index ()],
+ expr_id))
+ return false;
+ }
+ if (!has_avl_out)
+ return false;
+ }
+
+ for (insn_info *i = next_insn; i != next_insn->bb ()->head_insn ();
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && def_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (def_avl_p (i, info))
+ return false;
+ }
+
+ for (insn_info *i = prev_insn->bb ()->end_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && def_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (def_avl_p (i, info))
+ return false;
+ }
+ }
+ return true;
+ }
+
+ bool compatible_sew_lmul_with (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS \
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS) \
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool available_sew_lmul_with (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS \
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS) \
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_sew_lmul_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS \
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS) \
+ { \
+ gcc_assert (COMPATIBLE_P (prev, next)); \
+ FUSE (prev, next); \
+ prev.set_sew_lmul_demand (sew_lmul_demand_type::NEW_FLAGS); \
+ return; \
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_policy_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS \
+ && next_flags == policy_demand_type::NEXT_FLAGS) \
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool available_policy_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS \
+ && next_flags == policy_demand_type::NEXT_FLAGS) \
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_policy_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS \
+ && next_flags == policy_demand_type::NEXT_FLAGS) \
+ { \
+ gcc_assert (COMPATIBLE_P (prev, next)); \
+ FUSE (prev, next); \
+ prev.set_policy_demand (policy_demand_type::NEW_FLAGS); \
+ return; \
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_avl_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS \
+ && next_flags == avl_demand_type::NEXT_FLAGS) \
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool available_avl_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS \
+ && next_flags == avl_demand_type::NEXT_FLAGS) \
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_avl_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS \
+ && next_flags == avl_demand_type::NEXT_FLAGS) \
+ { \
+ gcc_assert (COMPATIBLE_P (prev, next)); \
+ FUSE (prev, next); \
+ prev.set_avl_demand (avl_demand_type::NEW_FLAGS); \
+ return; \
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ bool compatible_p = compatible_sew_lmul_with (prev, next)
+ && compatible_policy_with (prev, next)
+ && compatible_avl_with (prev, next);
+ return compatible_p;
+ }
+
+ bool available_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ bool available_p = available_sew_lmul_with (prev, next)
+ && available_policy_with (prev, next)
+ && available_avl_with (prev, next);
+ gcc_assert (!available_p || compatible_with (prev, next));
+ return available_p;
+ }
+
+ void merge_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (compatible_with (prev, next));
+ merge_sew_lmul_with (prev, next);
+ merge_policy_with (prev, next);
+ merge_avl_with (prev, next);
+ gcc_assert (available_with (prev, next));
+ }
+};
+
vector_infos_manager::vector_infos_manager ()
{
vector_edge_list = nullptr;
diff --git a/gcc/config/riscv/riscv-vsetvl.def b/gcc/config/riscv/riscv-vsetvl.def
index 709cc4ee0df..9bf804cc56b 100644
--- a/gcc/config/riscv/riscv-vsetvl.def
+++ b/gcc/config/riscv/riscv-vsetvl.def
@@ -18,496 +18,156 @@ You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
-#ifndef DEF_INCOMPATIBLE_COND
-#define DEF_INCOMPATIBLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, \
- GE_SEW1, TAIL_POLICTY1, MASK_POLICY1, AVL2, \
- SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, \
- TAIL_POLICTY2, MASK_POLICY2, COND)
+/* DEF_XXX_RULE (prev_demand, next_demand, fused_demand, compatible_p,
+ available_p, fuse)
+ prev_demand: the prev vector insn's sew_lmul_type
+ next_demand: the next vector insn's sew_lmul_type
+ fused_demand: if them are compatible, change prev_info demand to the
+ fused_demand after fuse prev_info and next_info
+ compatible_p: check if prev_demand and next_demand are compatible
+ available_p: check if prev_demand is available for next_demand
+ fuse: if them are compatible, how to modify prev_info */
+
+#ifndef DEF_SEW_LMUL_RULE
+#define DEF_SEW_LMUL_RULE(prev_demand, next_demand, fused_demand, \
+ compatible_p, available_p, fuse)
#endif
-#ifndef DEF_SEW_LMUL_FUSE_RULE
-#define DEF_SEW_LMUL_FUSE_RULE(DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, \
- DEMAND_GE_SEW1, DEMAND_SEW2, DEMAND_LMUL2, \
- DEMAND_RATIO2, DEMAND_GE_SEW2, NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, NEW_SEW, NEW_VLMUL, \
- NEW_RATIO)
+#ifndef DEF_POLICY_RULE
+#define DEF_POLICY_RULE(prev_demand, next_demand, fused_demand, compatible_p, \
+ available_p, fuse)
#endif
-#ifndef DEF_UNAVAILABLE_COND
-#define DEF_UNAVAILABLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, \
- TAIL_POLICTY1, MASK_POLICY1, AVL2, SEW2, LMUL2, \
- RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2, COND)
+#ifndef DEF_AVL_RULE
+#define DEF_AVL_RULE(prev_demand, next_demand, fused_demand, compatible_p, \
+ available_p, fuse)
#endif
-/* Case 1: Demand compatible AVL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ incompatible_avl_p)
-
-/* Case 2: Demand same SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_sew_p)
-
-/* Case 3: Demand same LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-
-/* Case 4: Demand same RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-
-/* Case 5: Demand same TAIL_POLICY. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_TRUE, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_TRUE, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_tail_policy_p)
-
-/* Case 6: Demand same MASK_POLICY. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_TRUE,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_TRUE,
- /*COND*/ different_mask_policy_p)
-
-/* Case 7: Demand non zero AVL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_ANY,
- DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ possible_zero_avl_p)
-
-/* Case 8: First SEW/LMUL/GE_SEW <-> Second RATIO/SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_invalid_for_first_sew_p)
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_invalid_for_first_lmul_p)
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_sew_less_than_first_sew_p)
-
-/* Case 9: First (GE_SEW + LMUL) <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_less_than_first_ratio_p)
-/* Case 11: First (SEW + LMUL) <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-/* Case 13: First (GE_SEW/SEW + RATIO) <-> Second LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-/* Case 14: First (LMUL + RATIO) <-> Second SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_sew_p)
-/* Case 15: First (LMUL + RATIO) <-> Second GE_SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ first_sew_less_than_second_sew_p)
-
-/* Case 16: First SEW + Second LMUL <-> First RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-/* Case 17: First SEW + Second LMUL <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_sew_p)
-
-/* Case 18: First SEW + Second RATIO <-> First LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-
-/* Case 19: First GE_SEW + Second LMUL <-> First RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_lmul_less_than_first_lmul_p)
-/* Case 20: First GE_SEW + Second LMUL <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_sew_less_than_first_sew_p)
-
-/* Case 21: First GE_SEW + Second RATIO <-> First LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_less_than_first_ratio_p)
-
-/* Case 22: First GE_SEW + Second SEW + First LMUL + Second ratio. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-
-/* Case 23: First GE_SEW + Second SEW + Second LMUL + First ratio. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-
-/* Merge rules. */
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ true, greatest_sew, first_vlmul,
- first_ratio)
-
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_ANY,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*SEW*/ DEMAND_ANY, /*LMUL*/ DEMAND_ANY,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_ANY,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, first_sew,
- vlmul_for_first_sew_second_ratio, second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_ANY, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_ANY,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, first_vlmul,
- ratio_for_second_sew_first_vlmul)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_FALSE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ true,
- /*NEW_DEMAND_GE_SEW*/ true, first_sew,
- vlmul_for_first_sew_second_ratio, second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_TRUE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ true,
- /*NEW_DEMAND_GE_SEW*/ true, greatest_sew,
- vlmul_for_greatest_sew_second_ratio, second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_FALSE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ true, first_sew, second_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, second_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, greatest_sew, second_vlmul,
- second_ratio)
-
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, second_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, first_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ true,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, first_vlmul,
- first_ratio)
-
-/* Define the unavailable cases for LCM. */
-
-/* Case 1: Dem1 (Not demand AVL) is unavailable to Dem2 (Demand AVL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_FALSE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-/* Case 2: Dem1 (Demand AVL) is unavailable to Dem2 (Demand normal AVL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ avl_unavailable_p)
-
-/* Case 3: Dem1 (Not demand TAIL) is unavailable to Dem2 (Demand TAIL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_FALSE, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_TRUE, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-
-/* Case 4: Dem1 (Not demand MASK) is unavailable to Dem2 (Demand MASK). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_FALSE,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_TRUE,
- /*COND*/ always_unavailable)
-
-/* Case 5: Dem1 (Demand RATIO) is unavailable to Dem2 (Demand SEW/GE_SEW/LMUL).
- */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_FALSE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_FALSE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-
-/* Case 6: Dem1 (Demand SEW). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ sew_unavailable_p)
-
-/* Case 7: Dem1 (Demand LMUL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_FALSE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ lmul_unavailable_p)
-
-/* Case 8: Dem1 (Demand GE_SEW). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ ge_sew_unavailable_p)
-
-/* Case 9: Dem1 (Demand GE_SEW + LMUL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ ge_sew_lmul_unavailable_p)
-
-/* Case 10: Dem1 (Demand GE_SEW + RATIO). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ ge_sew_ratio_unavailable_p)
-
-#undef DEF_INCOMPATIBLE_COND
-#undef DEF_SEW_LMUL_FUSE_RULE
-#undef DEF_UNAVAILABLE_COND
+/* Define SEW and LMUL rules. */
+DEF_SEW_LMUL_RULE (sew_lmul, sew_lmul, sew_lmul, eq_sew_lmul_p, eq_sew_lmul_p,
+ nop)
+DEF_SEW_LMUL_RULE (sew_lmul, ratio_only, sew_lmul, eq_ratio_p, eq_ratio_p, nop)
+DEF_SEW_LMUL_RULE (sew_lmul, sew_only, sew_lmul, eq_sew_p, eq_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_lmul, ge_sew, sew_lmul,
+ ge_next_sew_and_le_next_max_sew_p,
+ ge_next_sew_and_le_next_max_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_lmul, ratio_and_ge_sew, sew_lmul,
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p,
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p, nop)
+
+DEF_SEW_LMUL_RULE (ratio_only, sew_lmul, sew_lmul, eq_ratio_p, always_false,
+ use_next_sew_lmul)
+/* use_next_sew_lmul for testcase no change. */
+DEF_SEW_LMUL_RULE (ratio_only, ratio_only, ratio_only, eq_ratio_p, eq_ratio_p,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (ratio_only, sew_only, sew_lmul, has_prev_ratio_p,
+ always_false, use_next_sew_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_only, ge_sew, ratio_and_ge_sew, has_prev_ratio_p,
+ always_false, use_next_sew_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_only, ratio_and_ge_sew, ratio_and_ge_sew, eq_ratio_p,
+ always_false, use_next_sew_lmul)
+
+DEF_SEW_LMUL_RULE (sew_only, sew_lmul, sew_lmul, eq_sew_p, always_false,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (sew_only, ratio_only, sew_lmul, has_next_ratio_p,
+ always_false, modify_lmul_with_next_ratio)
+DEF_SEW_LMUL_RULE (sew_only, sew_only, sew_only, eq_sew_p, eq_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_only, ge_sew, sew_only,
+ ge_next_sew_and_le_next_max_sew_p, ge_next_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_only, ratio_and_ge_sew, sew_lmul,
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p,
+ always_false, modify_lmul_with_next_ratio)
+
+DEF_SEW_LMUL_RULE (ge_sew, sew_lmul, sew_lmul,
+ ge_prev_sew_and_le_prev_max_sew_p, always_false,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (ge_sew, ratio_only, ratio_and_ge_sew, has_next_ratio_p,
+ always_false, modify_lmul_with_next_ratio)
+DEF_SEW_LMUL_RULE (ge_sew, sew_only, sew_only,
+ ge_prev_sew_and_le_prev_max_sew_p, always_false,
+ use_next_sew)
+DEF_SEW_LMUL_RULE (ge_sew, ge_sew, ge_sew, max_sew_overlap_p, ge_next_sew_p,
+ use_max_sew)
+DEF_SEW_LMUL_RULE (ge_sew, ratio_and_ge_sew, ratio_and_ge_sew,
+ max_sew_overlap_and_has_next_ratio_p, always_false,
+ use_max_sew_and_lmul_with_next_ratio)
+
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, sew_lmul, sew_lmul,
+ ge_prev_sew_and_le_prev_max_sew_and_eq_ratio_p, always_false,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ratio_only, ratio_and_ge_sew, eq_ratio_p,
+ eq_ratio_p, use_max_sew_and_lmul_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, sew_only, sew_only,
+ ge_prev_sew_and_le_prev_max_sew_and_has_prev_ratio_p,
+ always_false, use_next_sew_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ge_sew, ratio_and_ge_sew,
+ max_sew_overlap_and_has_prev_ratio_p, ge_next_sew_p,
+ use_max_sew_and_lmul_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ratio_and_ge_sew, ratio_and_ge_sew,
+ max_sew_overlap_and_eq_ratio_p, ge_next_sew_and_eq_ratio_p,
+ use_max_sew_and_lmul_with_prev_ratio)
+
+/* Define TAIL and MASK compatible and merge rules. */
+
+DEF_POLICY_RULE (tail_mask_policy, tail_mask_policy, tail_mask_policy,
+ comp_tail_mask_policy_p, eq_tail_mask_policy_p,
+ use_tail_mask_policy)
+DEF_POLICY_RULE (tail_mask_policy, tail_policy_only, tail_mask_policy,
+ comp_tail_policy_p, eq_tail_policy_p, use_tail_policy)
+DEF_POLICY_RULE (tail_mask_policy, mask_policy_only, tail_mask_policy,
+ comp_mask_policy_p, eq_mask_policy_p, use_mask_policy)
+DEF_POLICY_RULE (tail_mask_policy, ignore_policy, tail_mask_policy, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (tail_policy_only, tail_mask_policy, tail_mask_policy,
+ comp_tail_policy_p, always_false, use_mask_policy)
+DEF_POLICY_RULE (tail_policy_only, tail_policy_only, tail_policy_only,
+ comp_tail_policy_p, eq_tail_policy_p, use_tail_policy)
+DEF_POLICY_RULE (tail_policy_only, mask_policy_only, tail_mask_policy,
+ always_true, always_false, use_mask_policy)
+DEF_POLICY_RULE (tail_policy_only, ignore_policy, tail_policy_only, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (mask_policy_only, tail_mask_policy, tail_mask_policy,
+ comp_mask_policy_p, always_false, use_tail_policy)
+DEF_POLICY_RULE (mask_policy_only, tail_policy_only, tail_mask_policy,
+ always_true, always_false, use_tail_policy)
+DEF_POLICY_RULE (mask_policy_only, mask_policy_only, mask_policy_only,
+ comp_mask_policy_p, eq_mask_policy_p, use_mask_policy)
+DEF_POLICY_RULE (mask_policy_only, ignore_policy, mask_policy_only, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (ignore_policy, tail_mask_policy, tail_mask_policy, always_true,
+ always_false, use_tail_mask_policy)
+DEF_POLICY_RULE (ignore_policy, tail_policy_only, tail_policy_only, always_true,
+ always_false, use_tail_policy)
+DEF_POLICY_RULE (ignore_policy, mask_policy_only, mask_policy_only, always_true,
+ always_false, use_mask_policy)
+DEF_POLICY_RULE (ignore_policy, ignore_policy, ignore_policy, always_true,
+ always_true, nop)
+
+/* Define AVL compatible and merge rules. */
+
+DEF_AVL_RULE (avl, avl, avl, equal_avl_p, equal_avl_p, nop)
+DEF_AVL_RULE (avl, non_zero_avl, avl, equal_avl_or_prev_non_zero_avl_p,
+ equal_avl_or_prev_non_zero_avl_p, nop)
+DEF_AVL_RULE (avl, ignore_avl, avl, always_true, always_true, nop)
+
+DEF_AVL_RULE (non_zero_avl, avl, avl,
+ equal_avl_or_next_non_zero_avl_and_can_use_next_avl_p,
+ always_false, use_next_avl_when_not_equal)
+
+DEF_AVL_RULE (non_zero_avl, non_zero_avl, non_zero_avl, always_true,
+ always_true, nop)
+DEF_AVL_RULE (non_zero_avl, ignore_avl, non_zero_avl, always_true, always_true,
+ nop)
+
+DEF_AVL_RULE (ignore_avl, avl, avl, can_use_next_avl_p, always_false,
+ use_next_avl)
+DEF_AVL_RULE (ignore_avl, non_zero_avl, non_zero_avl, can_use_next_avl_p,
+ always_false, use_next_avl)
+DEF_AVL_RULE (ignore_avl, ignore_avl, ignore_avl, always_true, always_true, nop)
+
+#undef DEF_SEW_LMUL_RULE
+#undef DEF_POLICY_RULE
+#undef DEF_AVL_RULE
diff --git a/gcc/config/riscv/riscv-vsetvl.h b/gcc/config/riscv/riscv-vsetvl.h
index 9eab276190e..542b2aea625 100644
--- a/gcc/config/riscv/riscv-vsetvl.h
+++ b/gcc/config/riscv/riscv-vsetvl.h
@@ -40,33 +40,6 @@ enum emit_type
EMIT_AFTER,
};
-enum demand_type
-{
- DEMAND_AVL,
- DEMAND_SEW,
- DEMAND_LMUL,
- DEMAND_RATIO,
- DEMAND_NONZERO_AVL,
- DEMAND_GE_SEW,
- DEMAND_TAIL_POLICY,
- DEMAND_MASK_POLICY,
- NUM_DEMAND
-};
-
-enum demand_status
-{
- DEMAND_FALSE,
- DEMAND_TRUE,
- DEMAND_ANY,
-};
-
-enum fusion_type
-{
- INVALID_FUSION,
- VALID_AVL_FUSION,
- KILLED_AVL_FUSION
-};
-
enum def_type
{
REAL_SET = 1 << 0,
@@ -171,57 +144,5 @@ public:
void dump (FILE *) const;
};
-struct demands_pair
-{
- demand_status first[NUM_DEMAND];
- demand_status second[NUM_DEMAND];
- bool match_cond_p (const bool *dems1, const bool *dems2) const
- {
- for (unsigned i = 0; i < NUM_DEMAND; i++)
- {
- if (first[i] != DEMAND_ANY && first[i] != dems1[i])
- return false;
- if (second[i] != DEMAND_ANY && second[i] != dems2[i])
- return false;
- }
- return true;
- }
-};
-
-struct demands_cond
-{
- demands_pair pair;
- using CONDITION_TYPE
- = bool (*) (const vector_insn_info &, const vector_insn_info &);
- CONDITION_TYPE incompatible_p;
- bool dual_incompatible_p (const vector_insn_info &info1,
- const vector_insn_info &info2) const
- {
- return ((pair.match_cond_p (info1.get_demands (), info2.get_demands ())
- && incompatible_p (info1, info2))
- || (pair.match_cond_p (info2.get_demands (), info1.get_demands ())
- && incompatible_p (info2, info1)));
- }
-};
-
-struct demands_fuse_rule
-{
- demands_pair pair;
- bool demand_sew_p;
- bool demand_lmul_p;
- bool demand_ratio_p;
- bool demand_ge_sew_p;
-
- using NEW_SEW
- = unsigned (*) (const vector_insn_info &, const vector_insn_info &);
- using NEW_VLMUL
- = vlmul_type (*) (const vector_insn_info &, const vector_insn_info &);
- using NEW_RATIO
- = unsigned (*) (const vector_insn_info &, const vector_insn_info &);
- NEW_SEW new_sew;
- NEW_VLMUL new_vlmul;
- NEW_RATIO new_ratio;
-};
-
} // namespace riscv_vector
#endif
--
2.36.3
@@ -1091,401 +1091,6 @@ calculate_vlmul (unsigned int sew, unsigned int ratio)
return LMUL_RESERVED;
}
-static bool
-incompatible_avl_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return !info1.compatible_avl_p (info2) && !info2.compatible_avl_p (info1);
-}
-
-static bool
-different_sew_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_sew () != info2.get_sew ();
-}
-
-static bool
-different_lmul_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_vlmul () != info2.get_vlmul ();
-}
-
-static bool
-different_ratio_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return info1.get_ratio () != info2.get_ratio ();
-}
-
-static bool
-different_tail_policy_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_ta () != info2.get_ta ();
-}
-
-static bool
-different_mask_policy_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_ma () != info2.get_ma ();
-}
-
-static bool
-possible_zero_avl_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return !info1.has_non_zero_avl () || !info2.has_non_zero_avl ();
-}
-
-static bool
-second_ratio_invalid_for_first_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (info1.get_sew (), info2.get_ratio ())
- == LMUL_RESERVED;
-}
-
-static bool
-second_ratio_invalid_for_first_lmul_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_sew (info1.get_vlmul (), info2.get_ratio ()) == 0;
-}
-
-static bool
-float_insn_valid_sew_p (const vector_insn_info &info, unsigned int sew)
-{
- if (info.get_insn () && info.get_insn ()->is_real ()
- && get_attr_type (info.get_insn ()->rtl ()) == TYPE_VFMOVFV)
- {
- if (sew == 16)
- return TARGET_VECTOR_ELEN_FP_16;
- else if (sew == 32)
- return TARGET_VECTOR_ELEN_FP_32;
- else if (sew == 64)
- return TARGET_VECTOR_ELEN_FP_64;
- }
- return true;
-}
-
-static bool
-second_sew_less_than_first_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info2.get_sew () < info1.get_sew ()
- || !float_insn_valid_sew_p (info1, info2.get_sew ());
-}
-
-static bool
-first_sew_less_than_second_sew_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info1.get_sew () < info2.get_sew ()
- || !float_insn_valid_sew_p (info2, info1.get_sew ());
-}
-
-/* return 0 if LMUL1 == LMUL2.
- return -1 if LMUL1 < LMUL2.
- return 1 if LMUL1 > LMUL2. */
-static int
-compare_lmul (vlmul_type vlmul1, vlmul_type vlmul2)
-{
- if (vlmul1 == vlmul2)
- return 0;
-
- switch (vlmul1)
- {
- case LMUL_1:
- if (vlmul2 == LMUL_2 || vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_2:
- if (vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_4:
- if (vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_8:
- return -1;
- case LMUL_F2:
- if (vlmul2 == LMUL_1 || vlmul2 == LMUL_2 || vlmul2 == LMUL_4
- || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_F4:
- if (vlmul2 == LMUL_F2 || vlmul2 == LMUL_1 || vlmul2 == LMUL_2
- || vlmul2 == LMUL_4 || vlmul2 == LMUL_8)
- return 1;
- else
- return -1;
- case LMUL_F8:
- return 0;
- default:
- gcc_unreachable ();
- }
-}
-
-static bool
-second_lmul_less_than_first_lmul_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return compare_lmul (info2.get_vlmul (), info1.get_vlmul ()) == -1;
-}
-
-static bool
-second_ratio_less_than_first_ratio_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return info2.get_ratio () < info1.get_ratio ();
-}
-
-static CONSTEXPR const demands_cond incompatible_conds[] = {
-#define DEF_INCOMPATIBLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, \
- GE_SEW1, TAIL_POLICTY1, MASK_POLICY1, AVL2, \
- SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, \
- TAIL_POLICTY2, MASK_POLICY2, COND) \
- {{{AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, TAIL_POLICTY1, \
- MASK_POLICY1}, \
- {AVL2, SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2}}, \
- COND},
-#include "riscv-vsetvl.def"
-};
-
-static unsigned
-greatest_sew (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return std::max (info1.get_sew (), info2.get_sew ());
-}
-
-static unsigned
-first_sew (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_sew ();
-}
-
-static unsigned
-second_sew (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_sew ();
-}
-
-static vlmul_type
-first_vlmul (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_vlmul ();
-}
-
-static vlmul_type
-second_vlmul (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_vlmul ();
-}
-
-static unsigned
-first_ratio (const vector_insn_info &info1, const vector_insn_info &)
-{
- return info1.get_ratio ();
-}
-
-static unsigned
-second_ratio (const vector_insn_info &, const vector_insn_info &info2)
-{
- return info2.get_ratio ();
-}
-
-static vlmul_type
-vlmul_for_first_sew_second_ratio (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (info1.get_sew (), info2.get_ratio ());
-}
-
-static vlmul_type
-vlmul_for_greatest_sew_second_ratio (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_vlmul (MAX (info1.get_sew (), info2.get_sew ()),
- info2.get_ratio ());
-}
-
-static unsigned
-ratio_for_second_sew_first_vlmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- return calculate_ratio (info2.get_sew (), info1.get_vlmul ());
-}
-
-static CONSTEXPR const demands_fuse_rule fuse_rules[] = {
-#define DEF_SEW_LMUL_FUSE_RULE(DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, \
- DEMAND_GE_SEW1, DEMAND_SEW2, DEMAND_LMUL2, \
- DEMAND_RATIO2, DEMAND_GE_SEW2, NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, NEW_SEW, NEW_VLMUL, \
- NEW_RATIO) \
- {{{DEMAND_ANY, DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, DEMAND_ANY, \
- DEMAND_GE_SEW1, DEMAND_ANY, DEMAND_ANY}, \
- {DEMAND_ANY, DEMAND_SEW2, DEMAND_LMUL2, DEMAND_RATIO2, DEMAND_ANY, \
- DEMAND_GE_SEW2, DEMAND_ANY, DEMAND_ANY}}, \
- NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, \
- NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, \
- NEW_SEW, \
- NEW_VLMUL, \
- NEW_RATIO},
-#include "riscv-vsetvl.def"
-};
-
-static bool
-always_unavailable (const vector_insn_info &, const vector_insn_info &)
-{
- return true;
-}
-
-static bool
-avl_unavailable_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- return !info2.compatible_avl_p (info1.get_avl_info ());
-}
-
-static bool
-sew_unavailable_p (const vector_insn_info &info1, const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL) && !info2.demand_p (DEMAND_RATIO))
- {
- if (info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return info1.get_sew () != info2.get_sew ();
- }
- return true;
-}
-
-static bool
-lmul_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.get_vlmul () == info2.get_vlmul () && !info2.demand_p (DEMAND_SEW)
- && !info2.demand_p (DEMAND_RATIO))
- return false;
- return true;
-}
-
-static bool
-ge_sew_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL) && !info2.demand_p (DEMAND_RATIO)
- && info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return true;
-}
-
-static bool
-ge_sew_lmul_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_RATIO) && info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- return true;
-}
-
-static bool
-ge_sew_ratio_unavailable_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (!info2.demand_p (DEMAND_LMUL))
- {
- if (info2.demand_p (DEMAND_GE_SEW))
- return info1.get_sew () < info2.get_sew ();
- /* Demand GE_SEW should be available for non-demand SEW. */
- else if (!info2.demand_p (DEMAND_SEW))
- return false;
- }
- return true;
-}
-
-static CONSTEXPR const demands_cond unavailable_conds[] = {
-#define DEF_UNAVAILABLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, \
- TAIL_POLICTY1, MASK_POLICY1, AVL2, SEW2, LMUL2, \
- RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2, COND) \
- {{{AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, TAIL_POLICTY1, \
- MASK_POLICY1}, \
- {AVL2, SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2}}, \
- COND},
-#include "riscv-vsetvl.def"
-};
-
-static bool
-same_sew_lmul_demand_p (const bool *dems1, const bool *dems2)
-{
- return dems1[DEMAND_SEW] == dems2[DEMAND_SEW]
- && dems1[DEMAND_LMUL] == dems2[DEMAND_LMUL]
- && dems1[DEMAND_RATIO] == dems2[DEMAND_RATIO] && !dems1[DEMAND_GE_SEW]
- && !dems2[DEMAND_GE_SEW];
-}
-
-static bool
-propagate_avl_across_demands_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info2.demand_p (DEMAND_AVL))
- {
- if (info2.demand_p (DEMAND_NONZERO_AVL))
- return info1.demand_p (DEMAND_AVL)
- && !info1.demand_p (DEMAND_NONZERO_AVL) && info1.has_avl_reg ();
- }
- else
- return info1.demand_p (DEMAND_AVL) && info1.has_avl_reg ();
- return false;
-}
-
-static bool
-reg_available_p (const insn_info *insn, const vector_insn_info &info)
-{
- if (info.has_avl_reg () && !info.get_avl_source ())
- return false;
- insn_info *def_insn = info.get_avl_source ()->insn ();
- if (def_insn->bb () == insn->bb ())
- return before_p (def_insn, insn);
- else
- return dominated_by_p (CDI_DOMINATORS, insn->bb ()->cfg_bb (),
- def_insn->bb ()->cfg_bb ());
-}
-
-/* Return true if the instruction support relaxed compatible check. */
-static bool
-support_relaxed_compatible_p (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (fault_first_load_p (info1.get_insn ()->rtl ())
- && info2.demand_p (DEMAND_AVL) && info2.has_avl_reg ()
- && info2.get_avl_source () && info2.get_avl_source ()->insn ()->is_phi ())
- {
- hash_set<set_info *> sets
- = get_all_sets (info2.get_avl_source (), true, false, false);
- for (set_info *set : sets)
- {
- if (read_vl_insn_p (set->insn ()->rtl ()))
- {
- const insn_info *insn
- = get_backward_fault_first_load_insn (set->insn ());
- if (insn == info1.get_insn ())
- return info2.compatible_vtype_p (info1);
- }
- }
- }
- return false;
-}
-
/* Count the number of REGNO in RINSN. */
static int
count_regno_occurrences (rtx_insn *rinsn, unsigned int regno)
@@ -2118,6 +1723,673 @@ public:
}
};
+class demand_system
+{
+private:
+ sbitmap *m_avl_def_in;
+ sbitmap *m_avl_def_out;
+
+ /* predictors. */
+
+ inline bool always_true (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {
+ return true;
+ }
+ inline bool always_false (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {
+ return false;
+ }
+
+ /* predictors for sew and lmul */
+
+ inline bool eq_lmul_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_vlmul () == next.get_vlmul ();
+ }
+ inline bool eq_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ();
+ }
+ inline bool eq_sew_lmul_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return eq_lmul_p (prev, next) && eq_sew_p (prev, next);
+ }
+ inline bool ge_next_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (next.get_ta () && prev.get_sew () > next.get_sew ());
+ }
+ inline bool ge_prev_sew_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.get_sew () == next.get_sew ()
+ || (prev.get_ta () && prev.get_sew () < next.get_sew ());
+ }
+ inline bool le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_sew () <= next.get_max_sew ();
+ }
+ inline bool le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_sew () <= prev.get_max_sew ();
+ }
+ inline bool max_sew_overlap_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return !(prev.get_sew () > next.get_max_sew ()
+ || next.get_sew () > prev.get_max_sew ());
+ }
+ inline bool eq_ratio_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return prev.has_same_ratio (next);
+ }
+ inline bool has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ratio () >= (next.get_sew () / 8);
+ }
+ inline bool has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return next.get_ratio () >= (prev.get_sew () / 8);
+ }
+
+ inline bool ge_next_sew_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && eq_ratio_p (prev, next);
+ }
+ inline bool ge_next_sew_and_le_next_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && le_next_max_sew_p (prev, next);
+ }
+ inline bool
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_next_sew_p (prev, next) && le_next_max_sew_p (prev, next)
+ && has_next_ratio_p (prev, next);
+ }
+ inline bool ge_prev_sew_and_le_prev_max_sew_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_prev_sew_p (prev, next) && le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_has_next_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return has_next_ratio_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+ inline bool
+ ge_prev_sew_and_le_prev_max_sew_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_prev_sew_p (prev, next) && eq_ratio_p (prev, next)
+ && le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return has_prev_ratio_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+ inline bool
+ ge_prev_sew_and_le_prev_max_sew_and_has_prev_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return ge_prev_sew_p (prev, next) && has_prev_ratio_p (prev, next)
+ && le_prev_max_sew_p (prev, next);
+ }
+ inline bool max_sew_overlap_and_eq_ratio_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return eq_ratio_p (prev, next) && max_sew_overlap_p (prev, next);
+ }
+
+ /* predictors for tail and mask policy */
+
+ inline bool eq_tail_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ta () == next.get_ta ();
+ }
+ inline bool eq_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ma () == next.get_ma ();
+ }
+ inline bool eq_tail_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return eq_tail_policy_p (prev, next) && eq_mask_policy_p (prev, next);
+ }
+
+ inline bool comp_tail_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ta () || next.get_ta () || eq_tail_policy_p (prev, next);
+ }
+
+ inline bool comp_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return prev.get_ma () || next.get_ma () || eq_mask_policy_p (prev, next);
+ }
+
+ inline bool comp_tail_mask_policy_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return comp_tail_policy_p (prev, next) && comp_mask_policy_p (prev, next);
+ }
+
+ /* predictors for avl */
+
+ inline bool def_or_use_vl_p (insn_info *i, const vsetvl_info &info)
+ {
+ return info.has_reg_vl ()
+ && (find_access (i->uses (), REGNO (info.get_vl ()))
+ || find_access (i->defs (), REGNO (info.get_vl ())));
+ }
+ inline bool def_avl_p (insn_info *i, const vsetvl_info &info)
+ {
+ return info.has_reg_avl ()
+ && find_access (i->defs (), REGNO (info.get_avl ()));
+ }
+
+ inline bool def_reg_between (insn_info *prev_insn, insn_info *curr_insn,
+ unsigned regno)
+ {
+ gcc_assert (prev_insn->compare_with (curr_insn) < 0);
+ for (insn_info *i = curr_insn->prev_nondebug_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def of regno
+ if (find_access (i->defs (), regno))
+ return true;
+ }
+ return false;
+ }
+
+ inline bool same_reg_avl_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!prev.has_reg_avl () || !next.has_reg_avl ())
+ return false;
+
+ if (same_equiv_note_p (prev.get_avl_def (), next.get_avl_def ()))
+ return true;
+
+ if (REGNO (prev.get_avl ()) != REGNO (next.get_avl ()))
+ return false;
+
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return safe_move_avl_vl_p (prev_insn, next_insn, next, false);
+ }
+
+ inline bool equal_avl_p (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+
+ if (prev.get_ratio () != next.get_ratio ())
+ return false;
+
+ if (next.has_reg_vl () && next.use_by_non_rvv_insn_p ())
+ return false;
+
+ if (vector_config_insn_p (prev.get_insn ()->rtl ()) && next.get_avl_def ()
+ && next.get_avl_def ()->insn () == prev.get_insn ())
+ return true;
+
+ if (prev.get_read_vl_insn ())
+ {
+ if (!next.has_reg_avl () || !next.get_avl_def ())
+ return false;
+ insn_info *avl_def_insn = extract_single_source (next.get_avl_def ());
+ return avl_def_insn == prev.get_read_vl_insn ();
+ }
+
+ if (prev == next && prev.has_reg_avl ())
+ {
+ insn_info *insn = prev.get_insn ();
+ bb_info *bb = insn->bb ();
+ for (insn_info *i = insn; real_insn_and_same_bb_p (i, bb);
+ i = i->next_nondebug_insn ())
+ if (find_access (i->defs (), REGNO (prev.get_avl ())))
+ return false;
+ }
+
+ if (prev.has_vlmax_avl () && next.has_vlmax_avl ())
+ return true;
+ else if (prev.has_imm_avl () && next.has_imm_avl ())
+ return INTVAL (prev.get_avl ()) == INTVAL (next.get_avl ());
+ else if (prev.has_reg_vl () && next.has_reg_avl ()
+ && REGNO (prev.get_vl ()) == REGNO (next.get_avl ()))
+ {
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return safe_move_avl_vl_p (prev_insn, next_insn, next, false);
+ }
+ else if (prev.has_reg_avl () && next.has_reg_avl ())
+ return same_reg_avl_p (prev, next);
+
+ return false;
+ }
+ inline bool equal_avl_or_prev_non_zero_avl_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ return equal_avl_p (prev, next) || prev.has_non_zero_avl ();
+ }
+
+ inline bool can_use_next_avl_p (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ if (!next.has_reg_avl () && !next.has_reg_vl ())
+ return true;
+
+ insn_info *prev_insn = prev.get_insn ();
+ if (prev.get_bb () != prev_insn->bb ())
+ prev_insn = prev.get_bb ()->end_insn ();
+
+ insn_info *next_insn = next.get_insn ();
+ if (next.get_bb () != next_insn->bb ())
+ next_insn = next.get_bb ()->end_insn ();
+
+ return safe_move_avl_vl_p (prev_insn, next_insn, next);
+ }
+
+ inline bool equal_avl_or_next_non_zero_avl_and_can_use_next_avl_p (
+ const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ return equal_avl_p (prev, next)
+ || (next.has_non_zero_avl () && can_use_next_avl_p (prev, next));
+ }
+
+ /* modifiers */
+
+ inline void nop (const vsetvl_info &prev ATTRIBUTE_UNUSED,
+ const vsetvl_info &next ATTRIBUTE_UNUSED)
+ {}
+
+ /* modifiers for sew and lmul */
+
+ inline void use_min_max_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ prev.set_max_sew (MIN (prev.get_max_sew (), next.get_max_sew ()));
+ }
+ inline void use_next_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ prev.set_sew (next.get_sew ());
+ use_min_max_sew (prev, next);
+ }
+ inline void use_max_sew (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ auto max_sew = std::max (prev.get_sew (), next.get_sew ());
+ prev.set_sew (max_sew);
+ use_min_max_sew (prev, next);
+ }
+ inline void use_next_sew_lmul (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ use_next_sew (prev, next);
+ prev.set_vlmul (next.get_vlmul ());
+ prev.set_ratio (next.get_ratio ());
+ }
+ inline void use_next_sew_with_prev_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ use_next_sew (prev, next);
+ prev.set_vlmul (calculate_vlmul (next.get_sew (), prev.get_ratio ()));
+ }
+ inline void modify_lmul_with_next_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ prev.set_vlmul (calculate_vlmul (prev.get_sew (), next.get_ratio ()));
+ prev.set_ratio (next.get_ratio ());
+ }
+
+ inline void use_max_sew_and_lmul_with_next_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ prev.set_vlmul (calculate_vlmul (prev.get_sew (), next.get_ratio ()));
+ use_max_sew (prev, next);
+ prev.set_ratio (next.get_ratio ());
+ }
+
+ inline void use_max_sew_and_lmul_with_prev_ratio (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ auto max_sew = std::max (prev.get_sew (), next.get_sew ());
+ prev.set_vlmul (calculate_vlmul (max_sew, prev.get_ratio ()));
+ prev.set_sew (max_sew);
+ }
+
+ /* modifiers for tail and mask policy */
+
+ inline void use_tail_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!next.get_ta ())
+ prev.set_ta (next.get_ta ());
+ }
+ inline void use_mask_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ if (!next.get_ma ())
+ prev.set_ma (next.get_ma ());
+ }
+ inline void use_tail_mask_policy (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ use_tail_policy (prev, next);
+ use_mask_policy (prev, next);
+ }
+
+ /* modifiers for avl */
+
+ inline void use_next_avl (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (can_use_next_avl_p (prev, next));
+ prev.update_avl (next);
+ }
+
+ inline void use_next_avl_when_not_equal (vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ if (equal_avl_p (prev, next))
+ return;
+ gcc_assert (next.has_non_zero_avl ());
+ use_next_avl (prev, next);
+ }
+
+public:
+ demand_system () : m_avl_def_in (nullptr), m_avl_def_out (nullptr) {}
+
+ void set_avl_in_out_data (sbitmap *avl_def_in, sbitmap *avl_def_out)
+ {
+ m_avl_def_in = avl_def_in;
+ m_avl_def_out = avl_def_out;
+ }
+
+ /* Can we move vsetvl info between prev_insn and next_insn safe? */
+ bool safe_move_avl_vl_p (insn_info *prev_insn, insn_info *next_insn,
+ const vsetvl_info &info, bool ignore_vl = false)
+ {
+ gcc_assert ((ignore_vl && info.has_reg_avl ())
+ || (info.has_reg_avl () || info.has_reg_vl ()));
+
+ gcc_assert (!prev_insn->is_debug_insn () && !next_insn->is_debug_insn ());
+ if (prev_insn->bb () == next_insn->bb ()
+ && prev_insn->compare_with (next_insn) < 0)
+ {
+ for (insn_info *i = next_insn->prev_nondebug_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && def_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (def_avl_p (i, info))
+ return false;
+ }
+ return true;
+ }
+ else
+ {
+ if (!ignore_vl && info.has_reg_vl ())
+ {
+ bitmap live_out = df_get_live_out (prev_insn->bb ()->cfg_bb ());
+ if (bitmap_bit_p (live_out, REGNO (info.get_vl ())))
+ return false;
+ }
+
+ if (info.has_reg_avl () && m_avl_def_in && m_avl_def_out)
+ {
+ bool has_avl_out = false;
+ unsigned regno = REGNO (info.get_avl ());
+ unsigned expr_id;
+ sbitmap_iterator sbi;
+ EXECUTE_IF_SET_IN_BITMAP (m_avl_def_out[prev_insn->bb ()->index ()],
+ 0, expr_id, sbi)
+ {
+ if (get_regno (expr_id, last_basic_block_for_fn (cfun))
+ != regno)
+ continue;
+ has_avl_out = true;
+ if (!bitmap_bit_p (m_avl_def_in[next_insn->bb ()->index ()],
+ expr_id))
+ return false;
+ }
+ if (!has_avl_out)
+ return false;
+ }
+
+ for (insn_info *i = next_insn; i != next_insn->bb ()->head_insn ();
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && def_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (def_avl_p (i, info))
+ return false;
+ }
+
+ for (insn_info *i = prev_insn->bb ()->end_insn (); i != prev_insn;
+ i = i->prev_nondebug_insn ())
+ {
+ // no def amd use of vl
+ if (!ignore_vl && def_or_use_vl_p (i, info))
+ return false;
+
+ // no def of avl
+ if (def_avl_p (i, info))
+ return false;
+ }
+ }
+ return true;
+ }
+
+ bool compatible_sew_lmul_with (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS \
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS) \
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool available_sew_lmul_with (const vsetvl_info &prev,
+ const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS \
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS) \
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_sew_lmul_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ sew_lmul_demand_type prev_flags = prev.get_sew_lmul_demand ();
+ sew_lmul_demand_type next_flags = next.get_sew_lmul_demand ();
+#define DEF_SEW_LMUL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == sew_lmul_demand_type::PREV_FLAGS \
+ && next_flags == sew_lmul_demand_type::NEXT_FLAGS) \
+ { \
+ gcc_assert (COMPATIBLE_P (prev, next)); \
+ FUSE (prev, next); \
+ prev.set_sew_lmul_demand (sew_lmul_demand_type::NEW_FLAGS); \
+ return; \
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_policy_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS \
+ && next_flags == policy_demand_type::NEXT_FLAGS) \
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool available_policy_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS \
+ && next_flags == policy_demand_type::NEXT_FLAGS) \
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_policy_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ policy_demand_type prev_flags = prev.get_policy_demand ();
+ policy_demand_type next_flags = next.get_policy_demand ();
+#define DEF_POLICY_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == policy_demand_type::PREV_FLAGS \
+ && next_flags == policy_demand_type::NEXT_FLAGS) \
+ { \
+ gcc_assert (COMPATIBLE_P (prev, next)); \
+ FUSE (prev, next); \
+ prev.set_policy_demand (policy_demand_type::NEW_FLAGS); \
+ return; \
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_avl_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS \
+ && next_flags == avl_demand_type::NEXT_FLAGS) \
+ return COMPATIBLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool available_avl_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS \
+ && next_flags == avl_demand_type::NEXT_FLAGS) \
+ return AVAILABLE_P (prev, next);
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ void merge_avl_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (prev.valid_p () && next.valid_p ());
+ avl_demand_type prev_flags = prev.get_avl_demand ();
+ avl_demand_type next_flags = next.get_avl_demand ();
+#define DEF_AVL_RULE(PREV_FLAGS, NEXT_FLAGS, NEW_FLAGS, COMPATIBLE_P, \
+ AVAILABLE_P, FUSE) \
+ if (prev_flags == avl_demand_type::PREV_FLAGS \
+ && next_flags == avl_demand_type::NEXT_FLAGS) \
+ { \
+ gcc_assert (COMPATIBLE_P (prev, next)); \
+ FUSE (prev, next); \
+ prev.set_avl_demand (avl_demand_type::NEW_FLAGS); \
+ return; \
+ }
+
+#include "riscv-vsetvl.def"
+
+ gcc_unreachable ();
+ }
+
+ bool compatible_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ bool compatible_p = compatible_sew_lmul_with (prev, next)
+ && compatible_policy_with (prev, next)
+ && compatible_avl_with (prev, next);
+ return compatible_p;
+ }
+
+ bool available_with (const vsetvl_info &prev, const vsetvl_info &next)
+ {
+ bool available_p = available_sew_lmul_with (prev, next)
+ && available_policy_with (prev, next)
+ && available_avl_with (prev, next);
+ gcc_assert (!available_p || compatible_with (prev, next));
+ return available_p;
+ }
+
+ void merge_with (vsetvl_info &prev, const vsetvl_info &next)
+ {
+ gcc_assert (compatible_with (prev, next));
+ merge_sew_lmul_with (prev, next);
+ merge_policy_with (prev, next);
+ merge_avl_with (prev, next);
+ gcc_assert (available_with (prev, next));
+ }
+};
+
vector_infos_manager::vector_infos_manager ()
{
vector_edge_list = nullptr;
@@ -18,496 +18,156 @@ You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
-#ifndef DEF_INCOMPATIBLE_COND
-#define DEF_INCOMPATIBLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, \
- GE_SEW1, TAIL_POLICTY1, MASK_POLICY1, AVL2, \
- SEW2, LMUL2, RATIO2, NONZERO_AVL2, GE_SEW2, \
- TAIL_POLICTY2, MASK_POLICY2, COND)
+/* DEF_XXX_RULE (prev_demand, next_demand, fused_demand, compatible_p,
+ available_p, fuse)
+ prev_demand: the prev vector insn's sew_lmul_type
+ next_demand: the next vector insn's sew_lmul_type
+ fused_demand: if them are compatible, change prev_info demand to the
+ fused_demand after fuse prev_info and next_info
+ compatible_p: check if prev_demand and next_demand are compatible
+ available_p: check if prev_demand is available for next_demand
+ fuse: if them are compatible, how to modify prev_info */
+
+#ifndef DEF_SEW_LMUL_RULE
+#define DEF_SEW_LMUL_RULE(prev_demand, next_demand, fused_demand, \
+ compatible_p, available_p, fuse)
#endif
-#ifndef DEF_SEW_LMUL_FUSE_RULE
-#define DEF_SEW_LMUL_FUSE_RULE(DEMAND_SEW1, DEMAND_LMUL1, DEMAND_RATIO1, \
- DEMAND_GE_SEW1, DEMAND_SEW2, DEMAND_LMUL2, \
- DEMAND_RATIO2, DEMAND_GE_SEW2, NEW_DEMAND_SEW, \
- NEW_DEMAND_LMUL, NEW_DEMAND_RATIO, \
- NEW_DEMAND_GE_SEW, NEW_SEW, NEW_VLMUL, \
- NEW_RATIO)
+#ifndef DEF_POLICY_RULE
+#define DEF_POLICY_RULE(prev_demand, next_demand, fused_demand, compatible_p, \
+ available_p, fuse)
#endif
-#ifndef DEF_UNAVAILABLE_COND
-#define DEF_UNAVAILABLE_COND(AVL1, SEW1, LMUL1, RATIO1, NONZERO_AVL1, GE_SEW1, \
- TAIL_POLICTY1, MASK_POLICY1, AVL2, SEW2, LMUL2, \
- RATIO2, NONZERO_AVL2, GE_SEW2, TAIL_POLICTY2, \
- MASK_POLICY2, COND)
+#ifndef DEF_AVL_RULE
+#define DEF_AVL_RULE(prev_demand, next_demand, fused_demand, compatible_p, \
+ available_p, fuse)
#endif
-/* Case 1: Demand compatible AVL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ incompatible_avl_p)
-
-/* Case 2: Demand same SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_sew_p)
-
-/* Case 3: Demand same LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-
-/* Case 4: Demand same RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-
-/* Case 5: Demand same TAIL_POLICY. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_TRUE, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_TRUE, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_tail_policy_p)
-
-/* Case 6: Demand same MASK_POLICY. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_TRUE,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_TRUE,
- /*COND*/ different_mask_policy_p)
-
-/* Case 7: Demand non zero AVL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_ANY,
- DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ possible_zero_avl_p)
-
-/* Case 8: First SEW/LMUL/GE_SEW <-> Second RATIO/SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_invalid_for_first_sew_p)
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_invalid_for_first_lmul_p)
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_sew_less_than_first_sew_p)
-
-/* Case 9: First (GE_SEW + LMUL) <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_less_than_first_ratio_p)
-/* Case 11: First (SEW + LMUL) <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-/* Case 13: First (GE_SEW/SEW + RATIO) <-> Second LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-/* Case 14: First (LMUL + RATIO) <-> Second SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_sew_p)
-/* Case 15: First (LMUL + RATIO) <-> Second GE_SEW. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ first_sew_less_than_second_sew_p)
-
-/* Case 16: First SEW + Second LMUL <-> First RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-/* Case 17: First SEW + Second LMUL <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_sew_p)
-
-/* Case 18: First SEW + Second RATIO <-> First LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-
-/* Case 19: First GE_SEW + Second LMUL <-> First RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_lmul_less_than_first_lmul_p)
-/* Case 20: First GE_SEW + Second LMUL <-> Second RATIO. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_sew_less_than_first_sew_p)
-
-/* Case 21: First GE_SEW + Second RATIO <-> First LMUL. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ second_ratio_less_than_first_ratio_p)
-
-/* Case 22: First GE_SEW + Second SEW + First LMUL + Second ratio. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_lmul_p)
-
-/* Case 23: First GE_SEW + Second SEW + Second LMUL + First ratio. */
-DEF_INCOMPATIBLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ different_ratio_p)
-
-/* Merge rules. */
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ true, greatest_sew, first_vlmul,
- first_ratio)
-
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_ANY,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*SEW*/ DEMAND_ANY, /*LMUL*/ DEMAND_ANY,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_ANY,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, first_sew,
- vlmul_for_first_sew_second_ratio, second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_ANY, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_ANY,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_ANY,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, first_vlmul,
- ratio_for_second_sew_first_vlmul)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_FALSE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ true,
- /*NEW_DEMAND_GE_SEW*/ true, first_sew,
- vlmul_for_first_sew_second_ratio, second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_TRUE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ true,
- /*NEW_DEMAND_GE_SEW*/ true, greatest_sew,
- vlmul_for_greatest_sew_second_ratio, second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_FALSE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ true, first_sew, second_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, second_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, greatest_sew, second_vlmul,
- second_ratio)
-
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, second_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_TRUE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ true,
- /*NEW_DEMAND_RATIO*/ false,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, first_vlmul,
- second_ratio)
-DEF_SEW_LMUL_FUSE_RULE (/*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_TRUE, /*GE_SEW*/ DEMAND_TRUE,
- /*SEW*/ DEMAND_TRUE, /*LMUL*/ DEMAND_FALSE,
- /*RATIO*/ DEMAND_FALSE, /*GE_SEW*/ DEMAND_FALSE,
- /*NEW_DEMAND_SEW*/ true,
- /*NEW_DEMAND_LMUL*/ false,
- /*NEW_DEMAND_RATIO*/ true,
- /*NEW_DEMAND_GE_SEW*/ false, second_sew, first_vlmul,
- first_ratio)
-
-/* Define the unavailable cases for LCM. */
-
-/* Case 1: Dem1 (Not demand AVL) is unavailable to Dem2 (Demand AVL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_FALSE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-/* Case 2: Dem1 (Demand AVL) is unavailable to Dem2 (Demand normal AVL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_TRUE, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ avl_unavailable_p)
-
-/* Case 3: Dem1 (Not demand TAIL) is unavailable to Dem2 (Demand TAIL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_FALSE, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_TRUE, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-
-/* Case 4: Dem1 (Not demand MASK) is unavailable to Dem2 (Demand MASK). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_FALSE,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_TRUE,
- /*COND*/ always_unavailable)
-
-/* Case 5: Dem1 (Demand RATIO) is unavailable to Dem2 (Demand SEW/GE_SEW/LMUL).
- */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_FALSE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_FALSE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ always_unavailable)
-
-/* Case 6: Dem1 (Demand SEW). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ sew_unavailable_p)
-
-/* Case 7: Dem1 (Demand LMUL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_FALSE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_FALSE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ lmul_unavailable_p)
-
-/* Case 8: Dem1 (Demand GE_SEW). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ ge_sew_unavailable_p)
-
-/* Case 9: Dem1 (Demand GE_SEW + LMUL). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_TRUE, /*RATIO*/ DEMAND_FALSE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ ge_sew_lmul_unavailable_p)
-
-/* Case 10: Dem1 (Demand GE_SEW + RATIO). */
-DEF_UNAVAILABLE_COND (/*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_TRUE,
- /*LMUL*/ DEMAND_FALSE, /*RATIO*/ DEMAND_TRUE,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_TRUE,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*AVL*/ DEMAND_ANY, /*SEW*/ DEMAND_ANY,
- /*LMUL*/ DEMAND_ANY, /*RATIO*/ DEMAND_ANY,
- /*NONZERO_AVL*/ DEMAND_ANY, /*GE_SEW*/ DEMAND_ANY,
- /*TAIL_POLICTY*/ DEMAND_ANY, /*MASK_POLICY*/ DEMAND_ANY,
- /*COND*/ ge_sew_ratio_unavailable_p)
-
-#undef DEF_INCOMPATIBLE_COND
-#undef DEF_SEW_LMUL_FUSE_RULE
-#undef DEF_UNAVAILABLE_COND
+/* Define SEW and LMUL rules. */
+DEF_SEW_LMUL_RULE (sew_lmul, sew_lmul, sew_lmul, eq_sew_lmul_p, eq_sew_lmul_p,
+ nop)
+DEF_SEW_LMUL_RULE (sew_lmul, ratio_only, sew_lmul, eq_ratio_p, eq_ratio_p, nop)
+DEF_SEW_LMUL_RULE (sew_lmul, sew_only, sew_lmul, eq_sew_p, eq_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_lmul, ge_sew, sew_lmul,
+ ge_next_sew_and_le_next_max_sew_p,
+ ge_next_sew_and_le_next_max_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_lmul, ratio_and_ge_sew, sew_lmul,
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p,
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p, nop)
+
+DEF_SEW_LMUL_RULE (ratio_only, sew_lmul, sew_lmul, eq_ratio_p, always_false,
+ use_next_sew_lmul)
+/* use_next_sew_lmul for testcase no change. */
+DEF_SEW_LMUL_RULE (ratio_only, ratio_only, ratio_only, eq_ratio_p, eq_ratio_p,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (ratio_only, sew_only, sew_lmul, has_prev_ratio_p,
+ always_false, use_next_sew_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_only, ge_sew, ratio_and_ge_sew, has_prev_ratio_p,
+ always_false, use_next_sew_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_only, ratio_and_ge_sew, ratio_and_ge_sew, eq_ratio_p,
+ always_false, use_next_sew_lmul)
+
+DEF_SEW_LMUL_RULE (sew_only, sew_lmul, sew_lmul, eq_sew_p, always_false,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (sew_only, ratio_only, sew_lmul, has_next_ratio_p,
+ always_false, modify_lmul_with_next_ratio)
+DEF_SEW_LMUL_RULE (sew_only, sew_only, sew_only, eq_sew_p, eq_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_only, ge_sew, sew_only,
+ ge_next_sew_and_le_next_max_sew_p, ge_next_sew_p, nop)
+DEF_SEW_LMUL_RULE (sew_only, ratio_and_ge_sew, sew_lmul,
+ ge_next_sew_and_le_next_max_sew_and_has_next_ratio_p,
+ always_false, modify_lmul_with_next_ratio)
+
+DEF_SEW_LMUL_RULE (ge_sew, sew_lmul, sew_lmul,
+ ge_prev_sew_and_le_prev_max_sew_p, always_false,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (ge_sew, ratio_only, ratio_and_ge_sew, has_next_ratio_p,
+ always_false, modify_lmul_with_next_ratio)
+DEF_SEW_LMUL_RULE (ge_sew, sew_only, sew_only,
+ ge_prev_sew_and_le_prev_max_sew_p, always_false,
+ use_next_sew)
+DEF_SEW_LMUL_RULE (ge_sew, ge_sew, ge_sew, max_sew_overlap_p, ge_next_sew_p,
+ use_max_sew)
+DEF_SEW_LMUL_RULE (ge_sew, ratio_and_ge_sew, ratio_and_ge_sew,
+ max_sew_overlap_and_has_next_ratio_p, always_false,
+ use_max_sew_and_lmul_with_next_ratio)
+
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, sew_lmul, sew_lmul,
+ ge_prev_sew_and_le_prev_max_sew_and_eq_ratio_p, always_false,
+ use_next_sew_lmul)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ratio_only, ratio_and_ge_sew, eq_ratio_p,
+ eq_ratio_p, use_max_sew_and_lmul_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, sew_only, sew_only,
+ ge_prev_sew_and_le_prev_max_sew_and_has_prev_ratio_p,
+ always_false, use_next_sew_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ge_sew, ratio_and_ge_sew,
+ max_sew_overlap_and_has_prev_ratio_p, ge_next_sew_p,
+ use_max_sew_and_lmul_with_prev_ratio)
+DEF_SEW_LMUL_RULE (ratio_and_ge_sew, ratio_and_ge_sew, ratio_and_ge_sew,
+ max_sew_overlap_and_eq_ratio_p, ge_next_sew_and_eq_ratio_p,
+ use_max_sew_and_lmul_with_prev_ratio)
+
+/* Define TAIL and MASK compatible and merge rules. */
+
+DEF_POLICY_RULE (tail_mask_policy, tail_mask_policy, tail_mask_policy,
+ comp_tail_mask_policy_p, eq_tail_mask_policy_p,
+ use_tail_mask_policy)
+DEF_POLICY_RULE (tail_mask_policy, tail_policy_only, tail_mask_policy,
+ comp_tail_policy_p, eq_tail_policy_p, use_tail_policy)
+DEF_POLICY_RULE (tail_mask_policy, mask_policy_only, tail_mask_policy,
+ comp_mask_policy_p, eq_mask_policy_p, use_mask_policy)
+DEF_POLICY_RULE (tail_mask_policy, ignore_policy, tail_mask_policy, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (tail_policy_only, tail_mask_policy, tail_mask_policy,
+ comp_tail_policy_p, always_false, use_mask_policy)
+DEF_POLICY_RULE (tail_policy_only, tail_policy_only, tail_policy_only,
+ comp_tail_policy_p, eq_tail_policy_p, use_tail_policy)
+DEF_POLICY_RULE (tail_policy_only, mask_policy_only, tail_mask_policy,
+ always_true, always_false, use_mask_policy)
+DEF_POLICY_RULE (tail_policy_only, ignore_policy, tail_policy_only, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (mask_policy_only, tail_mask_policy, tail_mask_policy,
+ comp_mask_policy_p, always_false, use_tail_policy)
+DEF_POLICY_RULE (mask_policy_only, tail_policy_only, tail_mask_policy,
+ always_true, always_false, use_tail_policy)
+DEF_POLICY_RULE (mask_policy_only, mask_policy_only, mask_policy_only,
+ comp_mask_policy_p, eq_mask_policy_p, use_mask_policy)
+DEF_POLICY_RULE (mask_policy_only, ignore_policy, mask_policy_only, always_true,
+ always_true, nop)
+
+DEF_POLICY_RULE (ignore_policy, tail_mask_policy, tail_mask_policy, always_true,
+ always_false, use_tail_mask_policy)
+DEF_POLICY_RULE (ignore_policy, tail_policy_only, tail_policy_only, always_true,
+ always_false, use_tail_policy)
+DEF_POLICY_RULE (ignore_policy, mask_policy_only, mask_policy_only, always_true,
+ always_false, use_mask_policy)
+DEF_POLICY_RULE (ignore_policy, ignore_policy, ignore_policy, always_true,
+ always_true, nop)
+
+/* Define AVL compatible and merge rules. */
+
+DEF_AVL_RULE (avl, avl, avl, equal_avl_p, equal_avl_p, nop)
+DEF_AVL_RULE (avl, non_zero_avl, avl, equal_avl_or_prev_non_zero_avl_p,
+ equal_avl_or_prev_non_zero_avl_p, nop)
+DEF_AVL_RULE (avl, ignore_avl, avl, always_true, always_true, nop)
+
+DEF_AVL_RULE (non_zero_avl, avl, avl,
+ equal_avl_or_next_non_zero_avl_and_can_use_next_avl_p,
+ always_false, use_next_avl_when_not_equal)
+
+DEF_AVL_RULE (non_zero_avl, non_zero_avl, non_zero_avl, always_true,
+ always_true, nop)
+DEF_AVL_RULE (non_zero_avl, ignore_avl, non_zero_avl, always_true, always_true,
+ nop)
+
+DEF_AVL_RULE (ignore_avl, avl, avl, can_use_next_avl_p, always_false,
+ use_next_avl)
+DEF_AVL_RULE (ignore_avl, non_zero_avl, non_zero_avl, can_use_next_avl_p,
+ always_false, use_next_avl)
+DEF_AVL_RULE (ignore_avl, ignore_avl, ignore_avl, always_true, always_true, nop)
+
+#undef DEF_SEW_LMUL_RULE
+#undef DEF_POLICY_RULE
+#undef DEF_AVL_RULE
@@ -40,33 +40,6 @@ enum emit_type
EMIT_AFTER,
};
-enum demand_type
-{
- DEMAND_AVL,
- DEMAND_SEW,
- DEMAND_LMUL,
- DEMAND_RATIO,
- DEMAND_NONZERO_AVL,
- DEMAND_GE_SEW,
- DEMAND_TAIL_POLICY,
- DEMAND_MASK_POLICY,
- NUM_DEMAND
-};
-
-enum demand_status
-{
- DEMAND_FALSE,
- DEMAND_TRUE,
- DEMAND_ANY,
-};
-
-enum fusion_type
-{
- INVALID_FUSION,
- VALID_AVL_FUSION,
- KILLED_AVL_FUSION
-};
-
enum def_type
{
REAL_SET = 1 << 0,
@@ -171,57 +144,5 @@ public:
void dump (FILE *) const;
};
-struct demands_pair
-{
- demand_status first[NUM_DEMAND];
- demand_status second[NUM_DEMAND];
- bool match_cond_p (const bool *dems1, const bool *dems2) const
- {
- for (unsigned i = 0; i < NUM_DEMAND; i++)
- {
- if (first[i] != DEMAND_ANY && first[i] != dems1[i])
- return false;
- if (second[i] != DEMAND_ANY && second[i] != dems2[i])
- return false;
- }
- return true;
- }
-};
-
-struct demands_cond
-{
- demands_pair pair;
- using CONDITION_TYPE
- = bool (*) (const vector_insn_info &, const vector_insn_info &);
- CONDITION_TYPE incompatible_p;
- bool dual_incompatible_p (const vector_insn_info &info1,
- const vector_insn_info &info2) const
- {
- return ((pair.match_cond_p (info1.get_demands (), info2.get_demands ())
- && incompatible_p (info1, info2))
- || (pair.match_cond_p (info2.get_demands (), info1.get_demands ())
- && incompatible_p (info2, info1)));
- }
-};
-
-struct demands_fuse_rule
-{
- demands_pair pair;
- bool demand_sew_p;
- bool demand_lmul_p;
- bool demand_ratio_p;
- bool demand_ge_sew_p;
-
- using NEW_SEW
- = unsigned (*) (const vector_insn_info &, const vector_insn_info &);
- using NEW_VLMUL
- = vlmul_type (*) (const vector_insn_info &, const vector_insn_info &);
- using NEW_RATIO
- = unsigned (*) (const vector_insn_info &, const vector_insn_info &);
- NEW_SEW new_sew;
- NEW_VLMUL new_vlmul;
- NEW_RATIO new_ratio;
-};
-
} // namespace riscv_vector
#endif