[V2,01/14] RISC-V: P1: Refactor avl_info/vl_vtype_info/vector_insn_info
Checks
Commit Message
This sub-patch combine avl_info/vl_vtype_info/vector_insn_info to
a single class vsetvl_info.
gcc/ChangeLog:
* config/riscv/riscv-vsetvl.cc (avl_info::avl_info): Removed.
(avl_info::single_source_equal_p): Ditto.
(avl_info::multiple_source_equal_p): Ditto.
(avl_info::operator=): Ditto.
(avl_info::operator==): Ditto.
(avl_info::operator!=): Ditto.
(avl_info::has_non_zero_avl): Ditto.
(vl_vtype_info::vl_vtype_info): Ditto.
(vl_vtype_info::operator==): Ditto.
(vl_vtype_info::operator!=): Ditto.
(vl_vtype_info::same_avl_p): Ditto.
(vl_vtype_info::same_vtype_p): Ditto.
(enum demand_flags): New enum.
(vl_vtype_info::same_vlmax_p): Removed.
(vector_insn_info::operator>=): Ditto.
(enum class): New demand types.
(vector_insn_info::operator==): Ditto.
(vector_insn_info::parse_insn): Ditto.
(class vsetvl_info): New class.
(vector_insn_info::compatible_p): Removed.
(vector_insn_info::skip_avl_compatible_p): Ditto.
(vector_insn_info::compatible_avl_p): Ditto.
(vector_insn_info::compatible_vtype_p): Ditto.
(vector_insn_info::available_p): Ditto.
(vector_insn_info::fuse_avl): Ditto.
(vector_insn_info::fuse_sew_lmul): Ditto.
(vector_insn_info::fuse_tail_policy): Ditto.
(vector_insn_info::fuse_mask_policy): Ditto.
(vector_insn_info::local_merge): Ditto.
(vector_insn_info::global_merge): Ditto.
(vector_insn_info::get_avl_or_vl_reg): Ditto.
(vector_insn_info::update_fault_first_load_avl): Ditto.
(vlmul_to_str): Ditto.
(policy_to_str): Ditto.
(vector_insn_info::dump): Ditto.
* config/riscv/riscv-vsetvl.h (class avl_info): Ditto.
(struct vl_vtype_info): Ditto.
(class vector_insn_info): Ditto.
---
gcc/config/riscv/riscv-vsetvl.cc | 1315 ++++++++++++------------------
gcc/config/riscv/riscv-vsetvl.h | 261 ------
2 files changed, 515 insertions(+), 1061 deletions(-)
--
2.36.3
Comments
+ bool use_by_non_rvv_insn;+ use_by_non_rvv_insn (false)
Change it into m_used_by_non_rvv_insn;
+ bool use_by_non_rvv_insn_p () const { return use_by_non_rvv_insn; }
Change it into 'bool used_by_non_rvv_insn_p () const { return m_used_by_non_rvv_insn; }'
+ bool has_reg_avl () const
+ {
+ return m_avl && REG_P (m_avl) && !has_vlmax_avl ();
+ }
--> has_nonvlmax_reg_avl
+ bool is_in_origin_bb () const { return get_insn ()->bb () == get_bb (); }
--> insn_inside_bb_p ()
And add comment here:
/* The block of INSN isn't always same as the block of the VSETVL_INFO,
meaning we may have 'get_insn ()->bb () != get_bb ()'.
E.g. BB 2 (Empty) ---> BB 3 (VALID, has rvv insn 1)
BB 2 has empty VSETVL_INFO, wheras BB 3 has VSETVL_INFO that satisfies get_insn ()->bb () == get_bb ()
In earliest fusion, we may fuse bb 3 and bb 2 so that the 'get_bb ()' of BB2 VSETVL_INFO will be BB2 wheras
the 'get_insn ()' of BB2 VSETVL INFO will be the rvv insn 1 (which is located at BB3). */
+ bool change_vtype_only;
---> m_change_vtype_only
juzhe.zhong@rivai.ai
From: Lehua Ding
Date: 2023-10-17 19:34
To: gcc-patches
CC: juzhe.zhong; kito.cheng; rdapp.gcc; palmer; jeffreyalaw; lehua.ding
Subject: [PATCH V2 01/14] RISC-V: P1: Refactor avl_info/vl_vtype_info/vector_insn_info
This sub-patch combine avl_info/vl_vtype_info/vector_insn_info to
a single class vsetvl_info.
gcc/ChangeLog:
* config/riscv/riscv-vsetvl.cc (avl_info::avl_info): Removed.
(avl_info::single_source_equal_p): Ditto.
(avl_info::multiple_source_equal_p): Ditto.
(avl_info::operator=): Ditto.
(avl_info::operator==): Ditto.
(avl_info::operator!=): Ditto.
(avl_info::has_non_zero_avl): Ditto.
(vl_vtype_info::vl_vtype_info): Ditto.
(vl_vtype_info::operator==): Ditto.
(vl_vtype_info::operator!=): Ditto.
(vl_vtype_info::same_avl_p): Ditto.
(vl_vtype_info::same_vtype_p): Ditto.
(enum demand_flags): New enum.
(vl_vtype_info::same_vlmax_p): Removed.
(vector_insn_info::operator>=): Ditto.
(enum class): New demand types.
(vector_insn_info::operator==): Ditto.
(vector_insn_info::parse_insn): Ditto.
(class vsetvl_info): New class.
(vector_insn_info::compatible_p): Removed.
(vector_insn_info::skip_avl_compatible_p): Ditto.
(vector_insn_info::compatible_avl_p): Ditto.
(vector_insn_info::compatible_vtype_p): Ditto.
(vector_insn_info::available_p): Ditto.
(vector_insn_info::fuse_avl): Ditto.
(vector_insn_info::fuse_sew_lmul): Ditto.
(vector_insn_info::fuse_tail_policy): Ditto.
(vector_insn_info::fuse_mask_policy): Ditto.
(vector_insn_info::local_merge): Ditto.
(vector_insn_info::global_merge): Ditto.
(vector_insn_info::get_avl_or_vl_reg): Ditto.
(vector_insn_info::update_fault_first_load_avl): Ditto.
(vlmul_to_str): Ditto.
(policy_to_str): Ditto.
(vector_insn_info::dump): Ditto.
* config/riscv/riscv-vsetvl.h (class avl_info): Ditto.
(struct vl_vtype_info): Ditto.
(class vector_insn_info): Ditto.
---
gcc/config/riscv/riscv-vsetvl.cc | 1315 ++++++++++++------------------
gcc/config/riscv/riscv-vsetvl.h | 261 ------
2 files changed, 515 insertions(+), 1061 deletions(-)
diff --git a/gcc/config/riscv/riscv-vsetvl.cc b/gcc/config/riscv/riscv-vsetvl.cc
index 4b06d93e7f9..79ba8466556 100644
--- a/gcc/config/riscv/riscv-vsetvl.cc
+++ b/gcc/config/riscv/riscv-vsetvl.cc
@@ -1581,827 +1581,542 @@ vsetvl_dominated_by_p (const basic_block cfg_bb,
return true;
}
-avl_info::avl_info (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
-}
-
-avl_info::avl_info (rtx value_in, set_info *source_in)
- : m_value (value_in), m_source (source_in)
-{}
-
-bool
-avl_info::single_source_equal_p (const avl_info &other) const
-{
- set_info *set1 = m_source;
- set_info *set2 = other.get_source ();
- insn_info *insn1 = extract_single_source (set1);
- insn_info *insn2 = extract_single_source (set2);
- if (!insn1 || !insn2)
- return false;
- return source_equal_p (insn1, insn2);
-}
-
-bool
-avl_info::multiple_source_equal_p (const avl_info &other) const
-{
- /* When the def info is same in RTL_SSA namespace, it's safe
- to consider they are avl compatible. */
- if (m_source == other.get_source ())
- return true;
-
- /* We only consider handle PHI node. */
- if (!m_source->insn ()->is_phi () || !other.get_source ()->insn ()->is_phi ())
- return false;
-
- phi_info *phi1 = as_a<phi_info *> (m_source);
- phi_info *phi2 = as_a<phi_info *> (other.get_source ());
-
- if (phi1->is_degenerate () && phi2->is_degenerate ())
- {
- /* Degenerate PHI means the PHI node only have one input. */
-
- /* If both PHI nodes have the same single input in use list.
- We consider they are AVL compatible. */
- if (phi1->input_value (0) == phi2->input_value (0))
- return true;
- }
- /* TODO: We can support more optimization cases in the future. */
- return false;
-}
-
-avl_info &
-avl_info::operator= (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
- return *this;
-}
-
-bool
-avl_info::operator== (const avl_info &other) const
-{
- if (!m_value)
- return !other.get_value ();
- if (!other.get_value ())
- return false;
-
- if (GET_CODE (m_value) != GET_CODE (other.get_value ()))
- return false;
-
- /* Handle CONST_INT AVL. */
- if (CONST_INT_P (m_value))
- return INTVAL (m_value) == INTVAL (other.get_value ());
-
- /* Handle VLMAX AVL. */
- if (vlmax_avl_p (m_value))
- return vlmax_avl_p (other.get_value ());
- if (vlmax_avl_p (other.get_value ()))
- return false;
-
- /* If any source is undef value, we think they are not equal. */
- if (!m_source || !other.get_source ())
- return false;
-
- /* If both sources are single source (defined by a single real RTL)
- and their definitions are same. */
- if (single_source_equal_p (other))
- return true;
-
- return multiple_source_equal_p (other);
-}
-
-bool
-avl_info::operator!= (const avl_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-avl_info::has_non_zero_avl () const
-{
- if (has_avl_imm ())
- return INTVAL (get_value ()) > 0;
- if (has_avl_reg ())
- return vlmax_avl_p (get_value ());
- return false;
-}
-
-/* Initialize VL/VTYPE information. */
-vl_vtype_info::vl_vtype_info (avl_info avl_in, uint8_t sew_in,
- enum vlmul_type vlmul_in, uint8_t ratio_in,
- bool ta_in, bool ma_in)
- : m_avl (avl_in), m_sew (sew_in), m_vlmul (vlmul_in), m_ratio (ratio_in),
- m_ta (ta_in), m_ma (ma_in)
-{
- gcc_assert (valid_sew_p (m_sew) && "Unexpected SEW");
-}
-
-bool
-vl_vtype_info::operator== (const vl_vtype_info &other) const
-{
- return same_avl_p (other) && m_sew == other.get_sew ()
- && m_vlmul == other.get_vlmul () && m_ta == other.get_ta ()
- && m_ma == other.get_ma () && m_ratio == other.get_ratio ();
-}
-
-bool
-vl_vtype_info::operator!= (const vl_vtype_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-vl_vtype_info::same_avl_p (const vl_vtype_info &other) const
-{
- /* We need to compare both RTL and SET. If both AVL are CONST_INT.
- For example, const_int 3 and const_int 4, we need to compare
- RTL. If both AVL are REG and their REGNO are same, we need to
- compare SET. */
- return get_avl () == other.get_avl ()
- && get_avl_source () == other.get_avl_source ();
-}
-
-bool
-vl_vtype_info::same_vtype_p (const vl_vtype_info &other) const
-{
- return get_sew () == other.get_sew () && get_vlmul () == other.get_vlmul ()
- && get_ta () == other.get_ta () && get_ma () == other.get_ma ();
-}
+/* This flags indicates the minimum demand of the vl and vtype values by the
+ RVV instruction. For example, DEMAND_RATIO_P indicates that this RVV
+ instruction only needs the SEW/LMUL ratio to remain the same, and does not
+ require SEW and LMUL to be fixed.
+ Therefore, if the former RVV instruction needs DEMAND_RATIO_P and the latter
+ instruction needs DEMAND_SEW_LMUL_P and its SEW/LMUL is the same as that of
+ the former instruction, then we can make the minimu demand of the former
+ instruction strict to DEMAND_SEW_LMUL_P, and its required SEW and LMUL are
+ the SEW and LMUL of the latter instruction, and the vsetvl instruction
+ generated according to the new demand can also be used for the latter
+ instruction, so there is no need to insert a separate vsetvl instruction for
+ the latter instruction. */
+enum demand_flags : unsigned
+{
+ DEMAND_EMPTY_P = 0,
+ DEMAND_SEW_P = 1 << 0,
+ DEMAND_LMUL_P = 1 << 1,
+ DEMAND_RATIO_P = 1 << 2,
+ DEMAND_GE_SEW_P = 1 << 3,
+ DEMAND_TAIL_POLICY_P = 1 << 4,
+ DEMAND_MASK_POLICY_P = 1 << 5,
+ DEMAND_AVL_P = 1 << 6,
+ DEMAND_NON_ZERO_AVL_P = 1 << 7,
+};
-bool
-vl_vtype_info::same_vlmax_p (const vl_vtype_info &other) const
-{
- return get_ratio () == other.get_ratio ();
-}
+/* We split the demand information into three parts. They are sew and lmul
+ related (sew_lmul_demand_type), tail and mask policy related
+ (policy_demand_type) and avl related (avl_demand_type). Then we define three
+ interfaces avaiable_with, compatible_with and merge_with. avaiable_with is
+ used to determine whether the two vsetvl infos prev_info and next_info are
+ available or not. If prev_info is available for next_info, it means that the
+ RVV insn corresponding to next_info on the path from prev_info to next_info
+ can be used without inserting a separate vsetvl instruction. compatible_with
+ is used to determine whether prev_info is compatible with next_info, and if
+ so, merge_with can be used to merge the stricter demand information from
+ next_info into prev_info so that prev_info becomes available to next_info.
+ */
-/* Compare the compatibility between Dem1 and Dem2.
- If Dem1 > Dem2, Dem1 has bigger compatibility then Dem2
- meaning Dem1 is easier be compatible with others than Dem2
- or Dem2 is stricter than Dem1.
- For example, Dem1 (demand SEW + LMUL) > Dem2 (demand RATIO). */
-bool
-vector_insn_info::operator>= (const vector_insn_info &other) const
+enum class sew_lmul_demand_type : unsigned
{
- if (support_relaxed_compatible_p (*this, other))
- {
- unsigned array_size = sizeof (unavailable_conds) / sizeof (demands_cond);
- /* Bypass AVL unavailable cases. */
- for (unsigned i = 2; i < array_size; i++)
- if (unavailable_conds[i].pair.match_cond_p (this->get_demands (),
- other.get_demands ())
- && unavailable_conds[i].incompatible_p (*this, other))
- return false;
- return true;
- }
-
- if (!other.compatible_p (static_cast<const vl_vtype_info &> (*this)))
- return false;
- if (!this->compatible_p (static_cast<const vl_vtype_info &> (other)))
- return true;
-
- if (*this == other)
- return true;
-
- for (const auto &cond : unavailable_conds)
- if (cond.pair.match_cond_p (this->get_demands (), other.get_demands ())
- && cond.incompatible_p (*this, other))
- return false;
-
- return true;
-}
+ sew_lmul = demand_flags::DEMAND_SEW_P | demand_flags::DEMAND_LMUL_P,
+ ratio_only = demand_flags::DEMAND_RATIO_P,
+ sew_only = demand_flags::DEMAND_SEW_P,
+ ge_sew = demand_flags::DEMAND_GE_SEW_P,
+ ratio_and_ge_sew
+ = demand_flags::DEMAND_RATIO_P | demand_flags::DEMAND_GE_SEW_P,
+};
-bool
-vector_insn_info::operator== (const vector_insn_info &other) const
+enum class policy_demand_type : unsigned
{
- gcc_assert (!uninit_p () && !other.uninit_p ()
- && "Uninitialization should not happen");
-
- /* Empty is only equal to another Empty. */
- if (empty_p ())
- return other.empty_p ();
- if (other.empty_p ())
- return empty_p ();
-
- /* Unknown is only equal to another Unknown. */
- if (unknown_p ())
- return other.unknown_p ();
- if (other.unknown_p ())
- return unknown_p ();
-
- for (size_t i = 0; i < NUM_DEMAND; i++)
- if (m_demands[i] != other.demand_p ((enum demand_type) i))
- return false;
-
- /* We should consider different INSN demands as different
- expression. Otherwise, we will be doing incorrect vsetvl
- elimination. */
- if (m_insn != other.get_insn ())
- return false;
-
- if (!same_avl_p (other))
- return false;
-
- /* If the full VTYPE is valid, check that it is the same. */
- return same_vtype_p (other);
-}
+ tail_mask_policy
+ = demand_flags::DEMAND_TAIL_POLICY_P | demand_flags::DEMAND_MASK_POLICY_P,
+ tail_policy_only = demand_flags::DEMAND_TAIL_POLICY_P,
+ mask_policy_only = demand_flags::DEMAND_MASK_POLICY_P,
+ ignore_policy = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (rtx_insn *rinsn)
+enum class avl_demand_type : unsigned
{
- *this = vector_insn_info ();
- if (!NONDEBUG_INSN_P (rinsn))
- return;
- if (optimize == 0 && !has_vtype_op (rinsn))
- return;
- gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
- m_state = VALID;
- extract_insn_cached (rinsn);
- rtx avl = ::get_avl (rinsn);
- m_avl = avl_info (avl, nullptr);
- m_sew = ::get_sew (rinsn);
- m_vlmul = ::get_vlmul (rinsn);
- m_ta = tail_agnostic_p (rinsn);
- m_ma = mask_agnostic_p (rinsn);
-}
+ avl = demand_flags::DEMAND_AVL_P,
+ non_zero_avl = demand_flags::DEMAND_NON_ZERO_AVL_P,
+ ignore_avl = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (insn_info *insn)
+class vsetvl_info
{
- *this = vector_insn_info ();
-
- /* Return if it is debug insn for the consistency with optimize == 0. */
- if (insn->is_debug_insn ())
- return;
+private:
+ insn_info *m_insn;
+ bb_info *m_bb;
+ rtx m_avl;
+ rtx m_vl;
+ set_info *m_avl_def;
+ uint8_t m_sew;
+ uint8_t m_max_sew;
+ vlmul_type m_vlmul;
+ uint8_t m_ratio;
+ bool m_ta;
+ bool m_ma;
+
+ sew_lmul_demand_type m_sew_lmul_demand;
+ policy_demand_type m_policy_demand;
+ avl_demand_type m_avl_demand;
+
+ enum class state_type
+ {
+ UNINITIALIZED,
+ VALID,
+ UNKNOWN,
+ EMPTY,
+ };
+ state_type m_state;
+
+ bool m_ignore;
+ bool change_vtype_only;
+ insn_info *m_read_vl_insn;
+ bool use_by_non_rvv_insn;
- /* We set it as unknown since we don't what will happen in CALL or ASM. */
- if (insn->is_call () || insn->is_asm ())
- {
- set_unknown ();
+public:
+ vsetvl_info ()
+ : m_insn (nullptr), m_bb (nullptr), m_avl (NULL_RTX), m_vl (NULL_RTX),
+ m_avl_def (nullptr), m_sew (0), m_max_sew (0), m_vlmul (LMUL_RESERVED),
+ m_ratio (0), m_ta (false), m_ma (false),
+ m_sew_lmul_demand (sew_lmul_demand_type::sew_lmul),
+ m_policy_demand (policy_demand_type::tail_mask_policy),
+ m_avl_demand (avl_demand_type::avl), m_state (state_type::UNINITIALIZED),
+ m_ignore (false), change_vtype_only (false), m_read_vl_insn (nullptr),
+ use_by_non_rvv_insn (false)
+ {}
+
+ vsetvl_info (insn_info *insn) : vsetvl_info () { parse_insn (insn); }
+
+ vsetvl_info (rtx_insn *insn) : vsetvl_info () { parse_insn (insn); }
+
+ void set_avl (rtx avl) { m_avl = avl; }
+ void set_vl (rtx vl) { m_vl = vl; }
+ void set_avl_def (set_info *avl_def) { m_avl_def = avl_def; }
+ void set_sew (uint8_t sew) { m_sew = sew; }
+ void set_vlmul (vlmul_type vlmul) { m_vlmul = vlmul; }
+ void set_ratio (uint8_t ratio) { m_ratio = ratio; }
+ void set_ta (bool ta) { m_ta = ta; }
+ void set_ma (bool ma) { m_ma = ma; }
+ void set_ignore () { m_ignore = true; }
+ void set_bb (bb_info *bb) { m_bb = bb; }
+ void set_max_sew (uint8_t max_sew) { m_max_sew = max_sew; }
+ void set_change_vtype_only () { change_vtype_only = true; }
+ void set_read_vl_insn (insn_info *insn) { m_read_vl_insn = insn; }
+
+ rtx get_avl () const { return m_avl; }
+ rtx get_vl () const { return m_vl; }
+ set_info *get_avl_def () const { return m_avl_def; }
+ uint8_t get_sew () const { return m_sew; }
+ vlmul_type get_vlmul () const { return m_vlmul; }
+ uint8_t get_ratio () const { return m_ratio; }
+ bool get_ta () const { return m_ta; }
+ bool get_ma () const { return m_ma; }
+ insn_info *get_insn () const { return m_insn; }
+ bool ignore_p () const { return m_ignore; }
+ bb_info *get_bb () const { return m_bb; }
+ uint8_t get_max_sew () const { return m_max_sew; }
+ insn_info *get_read_vl_insn () const { return m_read_vl_insn; }
+ bool use_by_non_rvv_insn_p () const { return use_by_non_rvv_insn; }
+
+ bool has_imm_avl () const { return m_avl && CONST_INT_P (m_avl); }
+ bool has_vlmax_avl () const { return vlmax_avl_p (m_avl); }
+ bool has_reg_avl () const
+ {
+ return m_avl && REG_P (m_avl) && !has_vlmax_avl ();
+ }
+ bool has_non_zero_avl () const
+ {
+ if (has_imm_avl ())
+ return INTVAL (m_avl) > 0;
+ return has_vlmax_avl ();
+ }
+ bool has_reg_vl () const
+ {
+ gcc_assert (!m_vl || REG_P (m_vl));
+ return m_vl && REG_P (m_vl);
+ }
+ bool has_same_ratio (const vsetvl_info &other) const
+ {
+ return get_ratio () == other.get_ratio ();
+ }
+ bool is_in_origin_bb () const { return get_insn ()->bb () == get_bb (); }
+ void update_avl (const vsetvl_info &other)
+ {
+ m_avl = other.get_avl ();
+ m_vl = other.get_vl ();
+ m_avl_def = other.get_avl_def ();
+ }
+
+ bool uninit_p () const { return m_state == state_type::UNINITIALIZED; }
+ bool valid_p () const { return m_state == state_type::VALID; }
+ bool unknown_p () const { return m_state == state_type::UNKNOWN; }
+ bool empty_p () const { return m_state == state_type::EMPTY; }
+ bool change_vtype_only_p () const { return change_vtype_only; }
+
+ void set_valid () { m_state = state_type::VALID; }
+ void set_unknown () { m_state = state_type::UNKNOWN; }
+ void set_empty () { m_state = state_type::EMPTY; }
+
+ void set_sew_lmul_demand (sew_lmul_demand_type demand)
+ {
+ m_sew_lmul_demand = demand;
+ }
+ void set_policy_demand (policy_demand_type demand)
+ {
+ m_policy_demand = demand;
+ }
+ void set_avl_demand (avl_demand_type demand) { m_avl_demand = demand; }
+
+ sew_lmul_demand_type get_sew_lmul_demand () const
+ {
+ return m_sew_lmul_demand;
+ }
+ policy_demand_type get_policy_demand () const { return m_policy_demand; }
+ avl_demand_type get_avl_demand () const { return m_avl_demand; }
+
+ void normalize_demand (unsigned demand_flags)
+ {
+ switch (demand_flags
+ & (DEMAND_SEW_P | DEMAND_LMUL_P | DEMAND_RATIO_P | DEMAND_GE_SEW_P))
+ {
+ case (unsigned) sew_lmul_demand_type::sew_lmul:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_lmul;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::sew_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ge_sew;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_and_ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_and_ge_sew;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_TAIL_POLICY_P | DEMAND_MASK_POLICY_P))
+ {
+ case (unsigned) policy_demand_type::tail_mask_policy:
+ m_policy_demand = policy_demand_type::tail_mask_policy;
+ break;
+ case (unsigned) policy_demand_type::tail_policy_only:
+ m_policy_demand = policy_demand_type::tail_policy_only;
+ break;
+ case (unsigned) policy_demand_type::mask_policy_only:
+ m_policy_demand = policy_demand_type::mask_policy_only;
+ break;
+ case (unsigned) policy_demand_type::ignore_policy:
+ m_policy_demand = policy_demand_type::ignore_policy;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_AVL_P | DEMAND_NON_ZERO_AVL_P))
+ {
+ case (unsigned) avl_demand_type::avl:
+ m_avl_demand = avl_demand_type::avl;
+ break;
+ case (unsigned) avl_demand_type::non_zero_avl:
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ break;
+ case (unsigned) avl_demand_type::ignore_avl:
+ m_avl_demand = avl_demand_type::ignore_avl;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ }
+
+ void parse_insn (rtx_insn *rinsn)
+ {
+ if (!NONDEBUG_INSN_P (rinsn))
return;
- }
-
- /* If this is something that updates VL/VTYPE that we don't know about, set
- the state to unknown. */
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
- && (find_access (insn->defs (), VL_REGNUM)
- || find_access (insn->defs (), VTYPE_REGNUM)))
- {
- set_unknown ();
+ if (optimize == 0 && !has_vtype_op (rinsn))
return;
- }
-
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
- return;
-
- /* Warning: This function has to work on both the lowered (i.e. post
- emit_local_forward_vsetvls) and pre-lowering forms. The main implication
- of this is that it can't use the value of a SEW, VL, or Policy operand as
- they might be stale after lowering. */
- vl_vtype_info::operator= (get_vl_vtype_info (insn));
- m_insn = insn;
- m_state = VALID;
- if (vector_config_insn_p (insn->rtl ()))
- {
- m_demands[DEMAND_AVL] = true;
- m_demands[DEMAND_RATIO] = true;
+ gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
+ set_valid ();
+ extract_insn_cached (rinsn);
+ m_avl = ::get_avl (rinsn);
+ if (has_vlmax_avl () || vsetvl_insn_p (rinsn))
+ m_vl = ::get_vl (rinsn);
+ m_sew = ::get_sew (rinsn);
+ m_vlmul = ::get_vlmul (rinsn);
+ m_ta = tail_agnostic_p (rinsn);
+ m_ma = mask_agnostic_p (rinsn);
+ }
+
+ void parse_insn (insn_info *insn)
+ {
+ m_insn = insn;
+ m_bb = insn->bb ();
+ /* Return if it is debug insn for the consistency with optimize == 0. */
+ if (insn->is_debug_insn ())
return;
- }
-
- if (has_vl_op (insn->rtl ()))
- m_demands[DEMAND_AVL] = true;
-
- if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_RATIO] = true;
- else
- {
- /* TODO: By default, if it doesn't demand RATIO, we set it
- demand SEW && LMUL both. Some instructions may demand SEW
- only and ignore LMUL, will fix it later. */
- m_demands[DEMAND_SEW] = true;
- if (!ignore_vlmul_insn_p (insn->rtl ()))
- m_demands[DEMAND_LMUL] = true;
- }
-
- if (get_attr_ta (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_TAIL_POLICY] = true;
- if (get_attr_ma (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_MASK_POLICY] = true;
-
- if (vector_config_insn_p (insn->rtl ()))
- return;
-
- if (scalar_move_insn_p (insn->rtl ()))
- {
- if (m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
- if (m_ta)
- m_demands[DEMAND_GE_SEW] = true;
- }
-
- if (!m_avl.has_avl_reg () || vlmax_avl_p (get_avl ()) || !m_avl.get_source ())
- return;
- if (!m_avl.get_source ()->insn ()->is_real ()
- && !m_avl.get_source ()->insn ()->is_phi ())
- return;
-
- insn_info *def_insn = extract_single_source (m_avl.get_source ());
- if (!def_insn || !vsetvl_insn_p (def_insn->rtl ()))
- return;
-
- vector_insn_info new_info;
- new_info.parse_insn (def_insn);
- if (!same_vlmax_p (new_info) && !scalar_move_insn_p (insn->rtl ()))
- return;
-
- if (new_info.has_avl ())
- {
- if (new_info.has_avl_imm ())
- set_avl_info (avl_info (new_info.get_avl (), nullptr));
- else
- {
- if (vlmax_avl_p (new_info.get_avl ()))
- set_avl_info (avl_info (new_info.get_avl (), get_avl_source ()));
- else
- {
- /* Conservatively propagate non-VLMAX AVL of user vsetvl:
- 1. The user vsetvl should be same block with the rvv insn.
- 2. The user vsetvl is the only def insn of rvv insn.
- 3. The AVL is not modified between def-use chain.
- 4. The VL is only used by insn within EBB.
- */
- bool modified_p = false;
- for (insn_info *i = def_insn->next_nondebug_insn ();
- real_insn_and_same_bb_p (i, get_insn ()->bb ());
- i = i->next_nondebug_insn ())
- {
- /* Consider this following sequence:
-
- insn 1: vsetvli a5,a3,e8,mf4,ta,mu
- insn 2: vsetvli zero,a5,e32,m1,ta,ma
- ...
- vle32.v v1,0(a1)
- vsetvli a2,zero,e32,m1,ta,ma
- vadd.vv v1,v1,v1
- vsetvli zero,a5,e32,m1,ta,ma
- vse32.v v1,0(a0)
- ...
- insn 3: sub a3,a3,a5
- ...
-
- We can local AVL propagate "a3" from insn 1 to insn 2
- if no insns between insn 1 and insn 2 modify "a3 even
- though insn 3 modifies "a3".
- Otherwise, we can't perform local AVL propagation.
-
- Early break if we reach the insn 2. */
- if (!before_p (i, insn))
- break;
- if (find_access (i->defs (), REGNO (new_info.get_avl ())))
- {
- modified_p = true;
- break;
- }
- }
-
- bool has_live_out_use = false;
- for (use_info *use : m_avl.get_source ()->all_uses ())
- {
- if (use->is_live_out_use ())
- {
- has_live_out_use = true;
- break;
- }
- }
- if (!modified_p && !has_live_out_use
- && def_insn == m_avl.get_source ()->insn ()
- && m_insn->bb () == def_insn->bb ())
- set_avl_info (new_info.get_avl_info ());
- }
- }
- }
- if (scalar_move_insn_p (insn->rtl ()) && m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
-}
-
-bool
-vector_insn_info::compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
-
- for (const auto &cond : incompatible_conds)
- if (cond.dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::skip_avl_compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
- unsigned array_size = sizeof (incompatible_conds) / sizeof (demands_cond);
- /* Bypass AVL incompatible cases. */
- for (unsigned i = 1; i < array_size; i++)
- if (incompatible_conds[i].dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::compatible_avl_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other.get_avl_info ();
-}
-
-bool
-vector_insn_info::compatible_avl_p (const avl_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- gcc_assert (demand_p (DEMAND_AVL) && "Can't compare AVL undemand state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other;
-}
-
-bool
-vector_insn_info::compatible_vtype_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare VTYPE in unknown state");
- if (demand_p (DEMAND_SEW))
- {
- if (!demand_p (DEMAND_GE_SEW) && m_sew != other.get_sew ())
- return false;
- if (demand_p (DEMAND_GE_SEW) && m_sew > other.get_sew ())
- return false;
- }
- if (demand_p (DEMAND_LMUL) && m_vlmul != other.get_vlmul ())
- return false;
- if (demand_p (DEMAND_RATIO) && m_ratio != other.get_ratio ())
- return false;
- if (demand_p (DEMAND_TAIL_POLICY) && m_ta != other.get_ta ())
- return false;
- if (demand_p (DEMAND_MASK_POLICY) && m_ma != other.get_ma ())
- return false;
- return true;
-}
-
-/* Determine whether the vector instructions requirements represented by
- Require are compatible with the previous vsetvli instruction represented
- by this. INSN is the instruction whose requirements we're considering. */
-bool
-vector_insn_info::compatible_p (const vl_vtype_info &curr_info) const
-{
- gcc_assert (!uninit_p () && "Can't handle uninitialized info");
- if (empty_p ())
- return false;
-
- /* Nothing is compatible with Unknown. */
- if (unknown_p ())
- return false;
-
- /* If the instruction doesn't need an AVLReg and the SEW matches, consider
- it compatible. */
- if (!demand_p (DEMAND_AVL))
- if (m_sew == curr_info.get_sew ())
- return true;
-
- return compatible_avl_p (curr_info) && compatible_vtype_p (curr_info);
-}
-
-bool
-vector_insn_info::available_p (const vector_insn_info &other) const
-{
- return *this >= other;
-}
-
-void
-vector_insn_info::fuse_avl (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- set_insn (info1.get_insn ());
- if (info1.demand_p (DEMAND_AVL))
- {
- if (info1.demand_p (DEMAND_NONZERO_AVL))
- {
- if (info2.demand_p (DEMAND_AVL)
- && !info2.demand_p (DEMAND_NONZERO_AVL))
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_AVL, true);
- set_demand (DEMAND_NONZERO_AVL, false);
- return;
- }
- }
- set_avl_info (info1.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info1.demand_p (DEMAND_NONZERO_AVL));
- }
- else
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info2.demand_p (DEMAND_NONZERO_AVL));
- }
- set_demand (DEMAND_AVL,
- info1.demand_p (DEMAND_AVL) || info2.demand_p (DEMAND_AVL));
-}
-
-void
-vector_insn_info::fuse_sew_lmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- /* We need to fuse sew && lmul according to demand info:
-
- 1. GE_SEW.
- 2. SEW.
- 3. LMUL.
- 4. RATIO. */
- if (same_sew_lmul_demand_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, info2.demand_p (DEMAND_SEW));
- set_demand (DEMAND_LMUL, info2.demand_p (DEMAND_LMUL));
- set_demand (DEMAND_RATIO, info2.demand_p (DEMAND_RATIO));
- set_demand (DEMAND_GE_SEW, info2.demand_p (DEMAND_GE_SEW));
- set_sew (info2.get_sew ());
- set_vlmul (info2.get_vlmul ());
- set_ratio (info2.get_ratio ());
+ /* We set it as unknown since we don't what will happen in CALL or ASM. */
+ if (insn->is_call () || insn->is_asm ())
+ {
+ set_unknown ();
+ return;
+ }
+
+ /* If this is something that updates VL/VTYPE that we don't know about, set
+ the state to unknown. */
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
+ && (find_access (insn->defs (), VL_REGNUM)
+ || find_access (insn->defs (), VTYPE_REGNUM)))
+ {
+ set_unknown ();
+ return;
+ }
+
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
+ /* uninitialized */
return;
- }
- for (const auto &rule : fuse_rules)
- {
- if (rule.pair.match_cond_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info1, info2));
- set_vlmul (rule.new_vlmul (info1, info2));
- set_ratio (rule.new_ratio (info1, info2));
- return;
- }
- if (rule.pair.match_cond_p (info2.get_demands (), info1.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info2, info1));
- set_vlmul (rule.new_vlmul (info2, info1));
- set_ratio (rule.new_ratio (info2, info1));
- return;
- }
- }
- gcc_unreachable ();
-}
-void
-vector_insn_info::fuse_tail_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info1.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else if (info2.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info2.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else
- set_ta (get_default_ta ());
-}
-
-void
-vector_insn_info::fuse_mask_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info1.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else if (info2.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info2.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else
- set_ma (get_default_ma ());
-}
-
-vector_insn_info
-vector_insn_info::local_merge (const vector_insn_info &merge_info) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
- /* For local backward data flow, we always update INSN && AVL as the
- latest INSN and AVL so that we can keep track status of each INSN. */
- new_info.fuse_avl (merge_info, *this);
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-vector_insn_info
-vector_insn_info::global_merge (const vector_insn_info &merge_info,
- unsigned int bb_index) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
-
- /* For global data flow, we should keep original INSN and AVL if they
- valid since we should keep the life information of each block.
-
- For example:
- bb 0 -> bb 1.
- We should keep INSN && AVL of bb 1 since we will eventually emit
- vsetvl instruction according to INSN and AVL of bb 1. */
- new_info.fuse_avl (*this, merge_info);
- /* Recompute the AVL source whose block index is equal to BB_INDEX. */
- if (new_info.get_avl_source ()
- && new_info.get_avl_source ()->insn ()->is_phi ()
- && new_info.get_avl_source ()->bb ()->index () != bb_index)
- {
- hash_set<set_info *> sets
- = get_all_sets (new_info.get_avl_source (), true, true, true);
- new_info.set_avl_source (nullptr);
- bool can_find_set_p = false;
- set_info *first_set = nullptr;
- for (set_info *set : sets)
- {
- if (!first_set)
- first_set = set;
- if (set->bb ()->index () == bb_index)
- {
- gcc_assert (!can_find_set_p);
- new_info.set_avl_source (set);
- can_find_set_p = true;
- }
- }
- if (!can_find_set_p && sets.elements () == 1
- && first_set->insn ()->is_real ())
- new_info.set_avl_source (first_set);
- }
-
- /* Make sure VLMAX AVL always has a set_info the get VL. */
- if (vlmax_avl_p (new_info.get_avl ()))
- {
- if (this->get_avl_source ())
- new_info.set_avl_source (this->get_avl_source ());
- else
- {
- gcc_assert (merge_info.get_avl_source ());
- new_info.set_avl_source (merge_info.get_avl_source ());
- }
- }
-
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-/* Wrapper helps to return the AVL or VL operand for the
- vector_insn_info. Return AVL if the AVL is not VLMAX.
- Otherwise, return the VL operand. */
-rtx
-vector_insn_info::get_avl_or_vl_reg (void) const
-{
- gcc_assert (has_avl_reg ());
- if (!vlmax_avl_p (get_avl ()))
- return get_avl ();
-
- rtx_insn *rinsn = get_insn ()->rtl ();
- if (has_vl_op (rinsn) || vsetvl_insn_p (rinsn))
- {
- rtx vl = ::get_vl (rinsn);
- /* For VLMAX, we should make sure we get the
- REG to emit 'vsetvl VL,zero' since the 'VL'
- should be the REG according to RVV ISA. */
- if (REG_P (vl))
- return vl;
- }
-
- /* We always has avl_source if it is VLMAX AVL. */
- gcc_assert (get_avl_source ());
- return get_avl_reg_rtx ();
-}
+ set_valid ();
+
+ m_avl = ::get_avl (insn->rtl ());
+ if (m_avl)
+ {
+ if (vsetvl_insn_p (insn->rtl ()) || has_vlmax_avl ())
+ m_vl = ::get_vl (insn->rtl ());
+
+ if (has_reg_avl ())
+ m_avl_def = find_access (insn->uses (), REGNO (m_avl))->def ();
+ }
+
+ m_sew = ::get_sew (insn->rtl ());
+ m_vlmul = ::get_vlmul (insn->rtl ());
+ m_ratio = get_attr_ratio (insn->rtl ());
+ /* when get_attr_ratio is invalid, this kind of instructions
+ doesn't care about ratio. However, we still need this value
+ in demand info backward analysis. */
+ if (m_ratio == INVALID_ATTRIBUTE)
+ m_ratio = calculate_ratio (m_sew, m_vlmul);
+ m_ta = tail_agnostic_p (insn->rtl ());
+ m_ma = mask_agnostic_p (insn->rtl ());
+
+ /* If merge operand is undef value, we prefer agnostic. */
+ int merge_op_idx = get_attr_merge_op_idx (insn->rtl ());
+ if (merge_op_idx != INVALID_ATTRIBUTE
+ && satisfies_constraint_vu (recog_data.operand[merge_op_idx]))
+ {
+ m_ta = true;
+ m_ma = true;
+ }
+
+ /* Determine the demand info of the RVV insn. */
+ m_max_sew = get_max_int_sew ();
+ unsigned demand_flags = 0;
+ if (vector_config_insn_p (insn->rtl ()))
+ {
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ }
+ else
+ {
+ if (has_vl_op (insn->rtl ()))
+ {
+ if (scalar_move_insn_p (insn->rtl ()))
+ {
+ /* If the avl for vmv.s.x comes from the vsetvl instruction, we
+ don't know if the avl is non-zero, so it is set to
+ DEMAND_AVL_P for now. it may be corrected to
+ DEMAND_NON_ZERO_AVL_P later when more information is
+ available.
+ */
+ if (has_non_zero_avl ())
+ demand_flags |= demand_flags::DEMAND_NON_ZERO_AVL_P;
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
-bool
-vector_insn_info::update_fault_first_load_avl (insn_info *insn)
-{
- // Update AVL to vl-output of the fault first load.
- const insn_info *read_vl = get_forward_read_vl_insn (insn);
- if (read_vl)
- {
- rtx vl = SET_DEST (PATTERN (read_vl->rtl ()));
- def_info *def = find_access (read_vl->defs (), REGNO (vl));
- set_info *set = safe_dyn_cast<set_info *> (def);
- set_avl_info (avl_info (vl, set));
- set_insn (insn);
- return true;
- }
- return false;
-}
+ if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ else
+ {
+ if (scalar_move_insn_p (insn->rtl ()) && m_ta)
+ {
+ demand_flags |= demand_flags::DEMAND_GE_SEW_P;
+ m_max_sew = get_attr_type (insn->rtl ()) == TYPE_VFMOVFV
+ ? get_max_float_sew ()
+ : get_max_int_sew ();
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_SEW_P;
+
+ if (!ignore_vlmul_insn_p (insn->rtl ()))
+ demand_flags |= demand_flags::DEMAND_LMUL_P;
+ }
-static const char *
-vlmul_to_str (vlmul_type vlmul)
-{
- switch (vlmul)
- {
- case LMUL_1:
- return "m1";
- case LMUL_2:
- return "m2";
- case LMUL_4:
- return "m4";
- case LMUL_8:
- return "m8";
- case LMUL_RESERVED:
- return "INVALID LMUL";
- case LMUL_F8:
- return "mf8";
- case LMUL_F4:
- return "mf4";
- case LMUL_F2:
- return "mf2";
+ if (!m_ta)
+ demand_flags |= demand_flags::DEMAND_TAIL_POLICY_P;
+ if (!m_ma)
+ demand_flags |= demand_flags::DEMAND_MASK_POLICY_P;
+ }
+
+ normalize_demand (demand_flags);
+
+ /* Optimize AVL from the vsetvl instruction. */
+ insn_info *def_insn = extract_single_source (get_avl_def ());
+ if (def_insn && vsetvl_insn_p (def_insn->rtl ()))
+ {
+ vsetvl_info def_info = vsetvl_info (def_insn);
+ if ((scalar_move_insn_p (insn->rtl ())
+ || def_info.get_ratio () == get_ratio ())
+ && (def_info.has_vlmax_avl () || def_info.has_imm_avl ()))
+ {
+ update_avl (def_info);
+ if (scalar_move_insn_p (insn->rtl ()) && has_non_zero_avl ())
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ }
+ }
+
+ /* Determine if dest operand(vl) has been used by non-RVV instructions. */
+ if (has_reg_vl ())
+ {
+ const hash_set<use_info *> vl_uses
+ = get_all_real_uses (get_insn (), REGNO (get_vl ()));
+ for (use_info *use : vl_uses)
+ {
+ gcc_assert (use->insn ()->is_real ());
+ rtx_insn *rinsn = use->insn ()->rtl ();
+ if (!has_vl_op (rinsn)
+ || count_regno_occurrences (rinsn, REGNO (get_vl ())) != 1)
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ rtx avl = ::get_avl (rinsn);
+ if (!avl || REGNO (get_vl ()) != REGNO (avl))
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ }
+ }
- default:
+ /* Collect the read vl insn for the fault-only-first rvv loads. */
+ if (fault_first_load_p (insn->rtl ()))
+ {
+ for (insn_info *i = insn->next_nondebug_insn ();
+ i->bb () == insn->bb (); i = i->next_nondebug_insn ())
+ {
+ if (find_access (i->defs (), VL_REGNUM))
+ break;
+ if (i->rtl () && read_vl_insn_p (i->rtl ()))
+ {
+ m_read_vl_insn = i;
+ break;
+ }
+ }
+ }
+ }
+
+ bool operator== (const vsetvl_info &other) const
+ {
+ gcc_assert (!uninit_p () && !other.uninit_p ()
+ && "Uninitialization should not happen");
+
+ if (empty_p ())
+ return other.empty_p ();
+ if (unknown_p ())
+ return other.unknown_p ();
+
+ return get_insn () == other.get_insn () && get_bb () == other.get_bb ()
+ && get_avl () == other.get_avl () && get_vl () == other.get_vl ()
+ && get_avl_def () == other.get_avl_def ()
+ && get_sew () == other.get_sew ()
+ && get_vlmul () == other.get_vlmul () && get_ta () == other.get_ta ()
+ && get_ma () == other.get_ma ()
+ && get_avl_demand () == other.get_avl_demand ()
+ && get_sew_lmul_demand () == other.get_sew_lmul_demand ()
+ && get_policy_demand () == other.get_policy_demand ();
+ }
+
+ void dump (FILE *file, const char *indent = "") const
+ {
+ if (uninit_p ())
+ {
+ fprintf (file, "UNINITIALIZED.\n");
+ return;
+ }
+ else if (unknown_p ())
+ {
+ fprintf (file, "UNKNOWN.\n");
+ return;
+ }
+ else if (empty_p ())
+ {
+ fprintf (file, "EMPTY.\n");
+ return;
+ }
+ else if (valid_p ())
+ fprintf (file, "VALID (insn %u, bb %u)%s\n", get_insn ()->uid (),
+ get_bb ()->index (), ignore_p () ? " (ignore)" : "");
+ else
gcc_unreachable ();
- }
-}
-
-static const char *
-policy_to_str (bool agnostic_p)
-{
- return agnostic_p ? "agnostic" : "undisturbed";
-}
-void
-vector_insn_info::dump (FILE *file) const
-{
- fprintf (file, "[");
- if (uninit_p ())
- fprintf (file, "UNINITIALIZED,");
- else if (valid_p ())
- fprintf (file, "VALID,");
- else if (unknown_p ())
- fprintf (file, "UNKNOWN,");
- else if (empty_p ())
- fprintf (file, "EMPTY,");
- else
- fprintf (file, "DIRTY,");
-
- fprintf (file, "Demand field={%d(VL),", demand_p (DEMAND_AVL));
- fprintf (file, "%d(DEMAND_NONZERO_AVL),", demand_p (DEMAND_NONZERO_AVL));
- fprintf (file, "%d(SEW),", demand_p (DEMAND_SEW));
- fprintf (file, "%d(DEMAND_GE_SEW),", demand_p (DEMAND_GE_SEW));
- fprintf (file, "%d(LMUL),", demand_p (DEMAND_LMUL));
- fprintf (file, "%d(RATIO),", demand_p (DEMAND_RATIO));
- fprintf (file, "%d(TAIL_POLICY),", demand_p (DEMAND_TAIL_POLICY));
- fprintf (file, "%d(MASK_POLICY)}\n", demand_p (DEMAND_MASK_POLICY));
-
- fprintf (file, "AVL=");
- print_rtl_single (file, get_avl ());
- fprintf (file, "SEW=%d,", get_sew ());
- fprintf (file, "VLMUL=%s,", vlmul_to_str (get_vlmul ()));
- fprintf (file, "RATIO=%d,", get_ratio ());
- fprintf (file, "TAIL_POLICY=%s,", policy_to_str (get_ta ()));
- fprintf (file, "MASK_POLICY=%s", policy_to_str (get_ma ()));
- fprintf (file, "]\n");
-
- if (valid_p ())
- {
- if (get_insn ())
- {
- fprintf (file, "The real INSN=");
- print_rtl_single (file, get_insn ()->rtl ());
- }
- }
-}
+ fprintf (file, "%sDemand fields:", indent);
+ if (m_sew_lmul_demand == sew_lmul_demand_type::sew_lmul)
+ fprintf (file, " demand_sew_lmul");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_only)
+ fprintf (file, " demand_ratio_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::sew_only)
+ fprintf (file, " demand_sew_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ge_sew)
+ fprintf (file, " demand_ge_sew");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_and_ge_sew)
+ fprintf (file, " demand_ratio_and_ge_sew");
+
+ if (m_policy_demand == policy_demand_type::tail_mask_policy)
+ fprintf (file, " demand_tail_mask_policy");
+ else if (m_policy_demand == policy_demand_type::tail_policy_only)
+ fprintf (file, " demand_tail_policy_only");
+ else if (m_policy_demand == policy_demand_type::mask_policy_only)
+ fprintf (file, " demand_mask_policy_only");
+
+ if (m_avl_demand == avl_demand_type::avl)
+ fprintf (file, " demand_avl");
+ else if (m_avl_demand == avl_demand_type::non_zero_avl)
+ fprintf (file, " demand_non_zero_avl");
+ fprintf (file, "\n");
+
+ fprintf (file, "%sSEW=%d, ", indent, get_sew ());
+ fprintf (file, "VLMUL=%s, ", vlmul_to_str (get_vlmul ()));
+ fprintf (file, "RATIO=%d, ", get_ratio ());
+ fprintf (file, "MAX_SEW=%d\n", get_max_sew ());
+
+ fprintf (file, "%sTAIL_POLICY=%s, ", indent, policy_to_str (get_ta ()));
+ fprintf (file, "MASK_POLICY=%s\n", policy_to_str (get_ma ()));
+
+ fprintf (file, "%sAVL=", indent);
+ print_rtl_single (file, get_avl ());
+ fprintf (file, "%sVL=", indent);
+ print_rtl_single (file, get_vl ());
+ if (change_vtype_only_p ())
+ fprintf (file, "%schange vtype only\n", indent);
+ if (get_read_vl_insn ())
+ fprintf (file, "%sread_vl_insn: insn %u\n", indent,
+ get_read_vl_insn ()->uid ());
+ if (use_by_non_rvv_insn_p ())
+ fprintf (file, "%suse_by_non_rvv_insn=true\n", indent);
+ }
+};
vector_infos_manager::vector_infos_manager ()
{
diff --git a/gcc/config/riscv/riscv-vsetvl.h b/gcc/config/riscv/riscv-vsetvl.h
index 53549abfac5..9eab276190e 100644
--- a/gcc/config/riscv/riscv-vsetvl.h
+++ b/gcc/config/riscv/riscv-vsetvl.h
@@ -82,267 +82,6 @@ enum def_type
CLOBBER_DEF = 1 << 4
};
-/* AVL info for RVV instruction. Most RVV instructions have AVL operand in
- implicit dependency. The AVL comparison between 2 RVV instructions is
- very important since it affects our decision whether we should insert
- a vsetvl instruction in this situation. AVL operand of all RVV instructions
- can only be either a const_int value with < 32 or a reg value which can be
- define by either a real RTL instruction or a PHI instruction. So we need a
- standalone method to define AVL comparison and we can not simpily use
- operator "==" to compare 2 RTX value since it's to strict which will make
- use miss a lot of optimization opportunities. This method handle these
- following cases:
-
- - Background:
- Insert-vsetvl PASS is working after RA.
-
- - Terminology:
- - pr: Pseudo-register.
- - hr: Hardware-register.
-
- - Case 1:
-
- Before RA:
- li pr138,13
- insn1 (implicit depend on pr138).
- li pr138,14
- insn2 (implicit depend on pr139).
-
- After RA:
- li hr5,13
- insn1 (implicit depend on hr5).
- li hr5,14
- insn2 (implicit depend on hr5).
-
- Correct IR after vsetvl PASS:
- li hr5,13
- vsetvl1 zero,hr5....
- insn1 (implicit depend on hr5).
- li hr5,14
- vsetvl2 zero,hr5....
- insn2 (implicit depend on hr5).
-
- In this case, both insn1 and insn2 are using hr5 as the same AVL.
- If we use "rtx_equal_p" or "REGNO (AVL1) == REGNO (AVL)", we will end
- up with missing the vsetvl2 instruction which creates wrong result.
-
- Note: Using "==" operator to compare 2 AVL RTX strictly can fix this
- issue. However, it is a too strict comparison method since not all member
- variables in RTX data structure are not neccessary to be the same. It will
- make us miss a lot of optimization opportunities.
-
- - Case 2:
-
- After RA:
- bb 0:
- li hr5,13
- bb 1:
- li hr5,14
- bb2:
- insn1 (implicit depend on hr5).
- insn2 (implicit depend on hr5).
-
- In this case, we may end up with different AVL RTX and produce redundant
- vsetvl instruction.
-
- VALUE is the implicit dependency in each RVV instruction.
- SOURCE is the source definition information of AVL operand. */
-class avl_info
-{
-private:
- rtx m_value;
- rtl_ssa::set_info *m_source;
-
-public:
- avl_info () : m_value (NULL_RTX), m_source (nullptr) {}
- avl_info (const avl_info &);
- avl_info (rtx, rtl_ssa::set_info *);
- rtx get_value () const { return m_value; }
- rtl_ssa::set_info *get_source () const { return m_source; }
- void set_source (rtl_ssa::set_info *set) { m_source = set; }
- bool single_source_equal_p (const avl_info &) const;
- bool multiple_source_equal_p (const avl_info &) const;
- avl_info &operator= (const avl_info &);
- bool operator== (const avl_info &) const;
- bool operator!= (const avl_info &) const;
-
- bool has_avl_imm () const
- {
- return get_value () && CONST_INT_P (get_value ());
- }
- bool has_avl_reg () const { return get_value () && REG_P (get_value ()); }
- bool has_avl_no_reg () const { return !get_value (); }
- bool has_non_zero_avl () const;
- bool has_avl () const { return get_value (); }
-};
-
-/* Basic structure to save VL/VTYPE information. */
-struct vl_vtype_info
-{
-protected:
- /* AVL can be either register or const_int. */
- avl_info m_avl;
- /* Fields from VTYPE. The VTYPE checking depend on the flag
- dem_* before. */
- uint8_t m_sew;
- riscv_vector::vlmul_type m_vlmul;
- uint8_t m_ratio;
- bool m_ta;
- bool m_ma;
-
-public:
- void set_sew (uint8_t sew) { m_sew = sew; }
- void set_vlmul (riscv_vector::vlmul_type vlmul) { m_vlmul = vlmul; }
- void set_ratio (uint8_t ratio) { m_ratio = ratio; }
- void set_ta (bool ta) { m_ta = ta; }
- void set_ma (bool ma) { m_ma = ma; }
-
- vl_vtype_info ()
- : m_avl (avl_info ()), m_sew (0), m_vlmul (riscv_vector::LMUL_RESERVED),
- m_ratio (0), m_ta (0), m_ma (0)
- {}
- vl_vtype_info (const vl_vtype_info &) = default;
- vl_vtype_info &operator= (const vl_vtype_info &) = default;
- vl_vtype_info (avl_info, uint8_t, riscv_vector::vlmul_type, uint8_t, bool,
- bool);
-
- bool operator== (const vl_vtype_info &) const;
- bool operator!= (const vl_vtype_info &) const;
-
- bool has_avl_imm () const { return m_avl.has_avl_imm (); }
- bool has_avl_reg () const { return m_avl.has_avl_reg (); }
- bool has_avl_no_reg () const { return m_avl.has_avl_no_reg (); }
- bool has_non_zero_avl () const { return m_avl.has_non_zero_avl (); };
- bool has_avl () const { return m_avl.has_avl (); }
-
- rtx get_avl () const { return m_avl.get_value (); }
- const avl_info &get_avl_info () const { return m_avl; }
- rtl_ssa::set_info *get_avl_source () const { return m_avl.get_source (); }
- void set_avl_source (rtl_ssa::set_info *set) { m_avl.set_source (set); }
- void set_avl_info (const avl_info &avl) { m_avl = avl; }
- uint8_t get_sew () const { return m_sew; }
- riscv_vector::vlmul_type get_vlmul () const { return m_vlmul; }
- uint8_t get_ratio () const { return m_ratio; }
- bool get_ta () const { return m_ta; }
- bool get_ma () const { return m_ma; }
-
- bool same_avl_p (const vl_vtype_info &) const;
- bool same_vtype_p (const vl_vtype_info &) const;
- bool same_vlmax_p (const vl_vtype_info &) const;
-};
-
-class vector_insn_info : public vl_vtype_info
-{
-private:
- enum state_type
- {
- UNINITIALIZED,
- VALID,
- UNKNOWN,
- EMPTY,
-
- /* The block is polluted as containing VSETVL instruction during dem
- backward propagation to gain better LCM optimization even though
- such VSETVL instruction is not really emit yet during this time. */
- DIRTY,
- };
-
- enum state_type m_state;
-
- bool m_demands[NUM_DEMAND];
-
- /* TODO: Assume INSN1 = INSN holding of definition of AVL.
- INSN2 = INSN that is inserted a vsetvl insn before.
- We may need to add a new member to save INSN of holding AVL.
- m_insn is holding the INSN that is inserted a vsetvl insn before in
- Phase 2. Ideally, most of the time INSN1 == INSN2. However, considering
- such case:
-
- vmv.x.s (INSN2)
- vle8.v (INSN1)
-
- If these 2 instructions are compatible, we should only issue a vsetvl INSN
- (with AVL included) before vmv.x.s, but vmv.x.s is not the INSN holding the
- definition of AVL. */
- rtl_ssa::insn_info *m_insn;
-
- friend class vector_infos_manager;
-
-public:
- vector_insn_info ()
- : vl_vtype_info (), m_state (UNINITIALIZED), m_demands{false},
- m_insn (nullptr)
- {}
-
- /* Parse the instruction to get VL/VTYPE information and demanding
- * information. */
- /* This is only called by simple_vsetvl subroutine when optimize == 0.
- Since RTL_SSA can not be enabled when optimize == 0, we don't initialize
- the m_insn. */
- void parse_insn (rtx_insn *);
- /* This is only called by lazy_vsetvl subroutine when optimize > 0.
- We use RTL_SSA framework to initialize the insn_info. */
- void parse_insn (rtl_ssa::insn_info *);
-
- bool operator>= (const vector_insn_info &) const;
- bool operator== (const vector_insn_info &) const;
-
- bool uninit_p () const { return m_state == UNINITIALIZED; }
- bool valid_p () const { return m_state == VALID; }
- bool unknown_p () const { return m_state == UNKNOWN; }
- bool empty_p () const { return m_state == EMPTY; }
- bool dirty_p () const { return m_state == DIRTY; }
- bool valid_or_dirty_p () const
- {
- return m_state == VALID || m_state == DIRTY;
- }
- bool available_p (const vector_insn_info &) const;
-
- static vector_insn_info get_unknown ()
- {
- vector_insn_info info;
- info.set_unknown ();
- return info;
- }
-
- void set_valid () { m_state = VALID; }
- void set_unknown () { m_state = UNKNOWN; }
- void set_empty () { m_state = EMPTY; }
- void set_dirty () { m_state = DIRTY; }
- void set_insn (rtl_ssa::insn_info *insn) { m_insn = insn; }
-
- bool demand_p (enum demand_type type) const { return m_demands[type]; }
- void demand (enum demand_type type) { m_demands[type] = true; }
- void set_demand (enum demand_type type, bool value)
- {
- m_demands[type] = value;
- }
- void fuse_avl (const vector_insn_info &, const vector_insn_info &);
- void fuse_sew_lmul (const vector_insn_info &, const vector_insn_info &);
- void fuse_tail_policy (const vector_insn_info &, const vector_insn_info &);
- void fuse_mask_policy (const vector_insn_info &, const vector_insn_info &);
-
- bool compatible_p (const vector_insn_info &) const;
- bool skip_avl_compatible_p (const vector_insn_info &) const;
- bool compatible_avl_p (const vl_vtype_info &) const;
- bool compatible_avl_p (const avl_info &) const;
- bool compatible_vtype_p (const vl_vtype_info &) const;
- bool compatible_p (const vl_vtype_info &) const;
- vector_insn_info local_merge (const vector_insn_info &) const;
- vector_insn_info global_merge (const vector_insn_info &, unsigned int) const;
-
- rtl_ssa::insn_info *get_insn () const { return m_insn; }
- const bool *get_demands (void) const { return m_demands; }
- rtx get_avl_or_vl_reg (void) const;
- rtx get_avl_reg_rtx (void) const
- {
- return gen_rtx_REG (Pmode, get_avl_source ()->regno ());
- }
- bool update_fault_first_load_avl (rtl_ssa::insn_info *);
-
- void dump (FILE *) const;
-};
-
struct vector_block_info
{
/* The local_dem vector insn_info of the block. */
--
2.36.3
+ bool has_reg_vl () const
+ {
+ gcc_assert (!m_vl || REG_P (m_vl));
+ return m_vl && REG_P (m_vl);
+ }
Add comment before the assertion:
/* The VL operand can only be either a NULL_RTX or a register. */
Rename has_reg_vl into has_vl since I don't think we need to specify the 'reg' for VL operand.
juzhe.zhong@rivai.ai
From: Lehua Ding
Date: 2023-10-17 19:34
To: gcc-patches
CC: juzhe.zhong; kito.cheng; rdapp.gcc; palmer; jeffreyalaw; lehua.ding
Subject: [PATCH V2 01/14] RISC-V: P1: Refactor avl_info/vl_vtype_info/vector_insn_info
This sub-patch combine avl_info/vl_vtype_info/vector_insn_info to
a single class vsetvl_info.
gcc/ChangeLog:
* config/riscv/riscv-vsetvl.cc (avl_info::avl_info): Removed.
(avl_info::single_source_equal_p): Ditto.
(avl_info::multiple_source_equal_p): Ditto.
(avl_info::operator=): Ditto.
(avl_info::operator==): Ditto.
(avl_info::operator!=): Ditto.
(avl_info::has_non_zero_avl): Ditto.
(vl_vtype_info::vl_vtype_info): Ditto.
(vl_vtype_info::operator==): Ditto.
(vl_vtype_info::operator!=): Ditto.
(vl_vtype_info::same_avl_p): Ditto.
(vl_vtype_info::same_vtype_p): Ditto.
(enum demand_flags): New enum.
(vl_vtype_info::same_vlmax_p): Removed.
(vector_insn_info::operator>=): Ditto.
(enum class): New demand types.
(vector_insn_info::operator==): Ditto.
(vector_insn_info::parse_insn): Ditto.
(class vsetvl_info): New class.
(vector_insn_info::compatible_p): Removed.
(vector_insn_info::skip_avl_compatible_p): Ditto.
(vector_insn_info::compatible_avl_p): Ditto.
(vector_insn_info::compatible_vtype_p): Ditto.
(vector_insn_info::available_p): Ditto.
(vector_insn_info::fuse_avl): Ditto.
(vector_insn_info::fuse_sew_lmul): Ditto.
(vector_insn_info::fuse_tail_policy): Ditto.
(vector_insn_info::fuse_mask_policy): Ditto.
(vector_insn_info::local_merge): Ditto.
(vector_insn_info::global_merge): Ditto.
(vector_insn_info::get_avl_or_vl_reg): Ditto.
(vector_insn_info::update_fault_first_load_avl): Ditto.
(vlmul_to_str): Ditto.
(policy_to_str): Ditto.
(vector_insn_info::dump): Ditto.
* config/riscv/riscv-vsetvl.h (class avl_info): Ditto.
(struct vl_vtype_info): Ditto.
(class vector_insn_info): Ditto.
---
gcc/config/riscv/riscv-vsetvl.cc | 1315 ++++++++++++------------------
gcc/config/riscv/riscv-vsetvl.h | 261 ------
2 files changed, 515 insertions(+), 1061 deletions(-)
diff --git a/gcc/config/riscv/riscv-vsetvl.cc b/gcc/config/riscv/riscv-vsetvl.cc
index 4b06d93e7f9..79ba8466556 100644
--- a/gcc/config/riscv/riscv-vsetvl.cc
+++ b/gcc/config/riscv/riscv-vsetvl.cc
@@ -1581,827 +1581,542 @@ vsetvl_dominated_by_p (const basic_block cfg_bb,
return true;
}
-avl_info::avl_info (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
-}
-
-avl_info::avl_info (rtx value_in, set_info *source_in)
- : m_value (value_in), m_source (source_in)
-{}
-
-bool
-avl_info::single_source_equal_p (const avl_info &other) const
-{
- set_info *set1 = m_source;
- set_info *set2 = other.get_source ();
- insn_info *insn1 = extract_single_source (set1);
- insn_info *insn2 = extract_single_source (set2);
- if (!insn1 || !insn2)
- return false;
- return source_equal_p (insn1, insn2);
-}
-
-bool
-avl_info::multiple_source_equal_p (const avl_info &other) const
-{
- /* When the def info is same in RTL_SSA namespace, it's safe
- to consider they are avl compatible. */
- if (m_source == other.get_source ())
- return true;
-
- /* We only consider handle PHI node. */
- if (!m_source->insn ()->is_phi () || !other.get_source ()->insn ()->is_phi ())
- return false;
-
- phi_info *phi1 = as_a<phi_info *> (m_source);
- phi_info *phi2 = as_a<phi_info *> (other.get_source ());
-
- if (phi1->is_degenerate () && phi2->is_degenerate ())
- {
- /* Degenerate PHI means the PHI node only have one input. */
-
- /* If both PHI nodes have the same single input in use list.
- We consider they are AVL compatible. */
- if (phi1->input_value (0) == phi2->input_value (0))
- return true;
- }
- /* TODO: We can support more optimization cases in the future. */
- return false;
-}
-
-avl_info &
-avl_info::operator= (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
- return *this;
-}
-
-bool
-avl_info::operator== (const avl_info &other) const
-{
- if (!m_value)
- return !other.get_value ();
- if (!other.get_value ())
- return false;
-
- if (GET_CODE (m_value) != GET_CODE (other.get_value ()))
- return false;
-
- /* Handle CONST_INT AVL. */
- if (CONST_INT_P (m_value))
- return INTVAL (m_value) == INTVAL (other.get_value ());
-
- /* Handle VLMAX AVL. */
- if (vlmax_avl_p (m_value))
- return vlmax_avl_p (other.get_value ());
- if (vlmax_avl_p (other.get_value ()))
- return false;
-
- /* If any source is undef value, we think they are not equal. */
- if (!m_source || !other.get_source ())
- return false;
-
- /* If both sources are single source (defined by a single real RTL)
- and their definitions are same. */
- if (single_source_equal_p (other))
- return true;
-
- return multiple_source_equal_p (other);
-}
-
-bool
-avl_info::operator!= (const avl_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-avl_info::has_non_zero_avl () const
-{
- if (has_avl_imm ())
- return INTVAL (get_value ()) > 0;
- if (has_avl_reg ())
- return vlmax_avl_p (get_value ());
- return false;
-}
-
-/* Initialize VL/VTYPE information. */
-vl_vtype_info::vl_vtype_info (avl_info avl_in, uint8_t sew_in,
- enum vlmul_type vlmul_in, uint8_t ratio_in,
- bool ta_in, bool ma_in)
- : m_avl (avl_in), m_sew (sew_in), m_vlmul (vlmul_in), m_ratio (ratio_in),
- m_ta (ta_in), m_ma (ma_in)
-{
- gcc_assert (valid_sew_p (m_sew) && "Unexpected SEW");
-}
-
-bool
-vl_vtype_info::operator== (const vl_vtype_info &other) const
-{
- return same_avl_p (other) && m_sew == other.get_sew ()
- && m_vlmul == other.get_vlmul () && m_ta == other.get_ta ()
- && m_ma == other.get_ma () && m_ratio == other.get_ratio ();
-}
-
-bool
-vl_vtype_info::operator!= (const vl_vtype_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-vl_vtype_info::same_avl_p (const vl_vtype_info &other) const
-{
- /* We need to compare both RTL and SET. If both AVL are CONST_INT.
- For example, const_int 3 and const_int 4, we need to compare
- RTL. If both AVL are REG and their REGNO are same, we need to
- compare SET. */
- return get_avl () == other.get_avl ()
- && get_avl_source () == other.get_avl_source ();
-}
-
-bool
-vl_vtype_info::same_vtype_p (const vl_vtype_info &other) const
-{
- return get_sew () == other.get_sew () && get_vlmul () == other.get_vlmul ()
- && get_ta () == other.get_ta () && get_ma () == other.get_ma ();
-}
+/* This flags indicates the minimum demand of the vl and vtype values by the
+ RVV instruction. For example, DEMAND_RATIO_P indicates that this RVV
+ instruction only needs the SEW/LMUL ratio to remain the same, and does not
+ require SEW and LMUL to be fixed.
+ Therefore, if the former RVV instruction needs DEMAND_RATIO_P and the latter
+ instruction needs DEMAND_SEW_LMUL_P and its SEW/LMUL is the same as that of
+ the former instruction, then we can make the minimu demand of the former
+ instruction strict to DEMAND_SEW_LMUL_P, and its required SEW and LMUL are
+ the SEW and LMUL of the latter instruction, and the vsetvl instruction
+ generated according to the new demand can also be used for the latter
+ instruction, so there is no need to insert a separate vsetvl instruction for
+ the latter instruction. */
+enum demand_flags : unsigned
+{
+ DEMAND_EMPTY_P = 0,
+ DEMAND_SEW_P = 1 << 0,
+ DEMAND_LMUL_P = 1 << 1,
+ DEMAND_RATIO_P = 1 << 2,
+ DEMAND_GE_SEW_P = 1 << 3,
+ DEMAND_TAIL_POLICY_P = 1 << 4,
+ DEMAND_MASK_POLICY_P = 1 << 5,
+ DEMAND_AVL_P = 1 << 6,
+ DEMAND_NON_ZERO_AVL_P = 1 << 7,
+};
-bool
-vl_vtype_info::same_vlmax_p (const vl_vtype_info &other) const
-{
- return get_ratio () == other.get_ratio ();
-}
+/* We split the demand information into three parts. They are sew and lmul
+ related (sew_lmul_demand_type), tail and mask policy related
+ (policy_demand_type) and avl related (avl_demand_type). Then we define three
+ interfaces avaiable_with, compatible_with and merge_with. avaiable_with is
+ used to determine whether the two vsetvl infos prev_info and next_info are
+ available or not. If prev_info is available for next_info, it means that the
+ RVV insn corresponding to next_info on the path from prev_info to next_info
+ can be used without inserting a separate vsetvl instruction. compatible_with
+ is used to determine whether prev_info is compatible with next_info, and if
+ so, merge_with can be used to merge the stricter demand information from
+ next_info into prev_info so that prev_info becomes available to next_info.
+ */
-/* Compare the compatibility between Dem1 and Dem2.
- If Dem1 > Dem2, Dem1 has bigger compatibility then Dem2
- meaning Dem1 is easier be compatible with others than Dem2
- or Dem2 is stricter than Dem1.
- For example, Dem1 (demand SEW + LMUL) > Dem2 (demand RATIO). */
-bool
-vector_insn_info::operator>= (const vector_insn_info &other) const
+enum class sew_lmul_demand_type : unsigned
{
- if (support_relaxed_compatible_p (*this, other))
- {
- unsigned array_size = sizeof (unavailable_conds) / sizeof (demands_cond);
- /* Bypass AVL unavailable cases. */
- for (unsigned i = 2; i < array_size; i++)
- if (unavailable_conds[i].pair.match_cond_p (this->get_demands (),
- other.get_demands ())
- && unavailable_conds[i].incompatible_p (*this, other))
- return false;
- return true;
- }
-
- if (!other.compatible_p (static_cast<const vl_vtype_info &> (*this)))
- return false;
- if (!this->compatible_p (static_cast<const vl_vtype_info &> (other)))
- return true;
-
- if (*this == other)
- return true;
-
- for (const auto &cond : unavailable_conds)
- if (cond.pair.match_cond_p (this->get_demands (), other.get_demands ())
- && cond.incompatible_p (*this, other))
- return false;
-
- return true;
-}
+ sew_lmul = demand_flags::DEMAND_SEW_P | demand_flags::DEMAND_LMUL_P,
+ ratio_only = demand_flags::DEMAND_RATIO_P,
+ sew_only = demand_flags::DEMAND_SEW_P,
+ ge_sew = demand_flags::DEMAND_GE_SEW_P,
+ ratio_and_ge_sew
+ = demand_flags::DEMAND_RATIO_P | demand_flags::DEMAND_GE_SEW_P,
+};
-bool
-vector_insn_info::operator== (const vector_insn_info &other) const
+enum class policy_demand_type : unsigned
{
- gcc_assert (!uninit_p () && !other.uninit_p ()
- && "Uninitialization should not happen");
-
- /* Empty is only equal to another Empty. */
- if (empty_p ())
- return other.empty_p ();
- if (other.empty_p ())
- return empty_p ();
-
- /* Unknown is only equal to another Unknown. */
- if (unknown_p ())
- return other.unknown_p ();
- if (other.unknown_p ())
- return unknown_p ();
-
- for (size_t i = 0; i < NUM_DEMAND; i++)
- if (m_demands[i] != other.demand_p ((enum demand_type) i))
- return false;
-
- /* We should consider different INSN demands as different
- expression. Otherwise, we will be doing incorrect vsetvl
- elimination. */
- if (m_insn != other.get_insn ())
- return false;
-
- if (!same_avl_p (other))
- return false;
-
- /* If the full VTYPE is valid, check that it is the same. */
- return same_vtype_p (other);
-}
+ tail_mask_policy
+ = demand_flags::DEMAND_TAIL_POLICY_P | demand_flags::DEMAND_MASK_POLICY_P,
+ tail_policy_only = demand_flags::DEMAND_TAIL_POLICY_P,
+ mask_policy_only = demand_flags::DEMAND_MASK_POLICY_P,
+ ignore_policy = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (rtx_insn *rinsn)
+enum class avl_demand_type : unsigned
{
- *this = vector_insn_info ();
- if (!NONDEBUG_INSN_P (rinsn))
- return;
- if (optimize == 0 && !has_vtype_op (rinsn))
- return;
- gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
- m_state = VALID;
- extract_insn_cached (rinsn);
- rtx avl = ::get_avl (rinsn);
- m_avl = avl_info (avl, nullptr);
- m_sew = ::get_sew (rinsn);
- m_vlmul = ::get_vlmul (rinsn);
- m_ta = tail_agnostic_p (rinsn);
- m_ma = mask_agnostic_p (rinsn);
-}
+ avl = demand_flags::DEMAND_AVL_P,
+ non_zero_avl = demand_flags::DEMAND_NON_ZERO_AVL_P,
+ ignore_avl = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (insn_info *insn)
+class vsetvl_info
{
- *this = vector_insn_info ();
-
- /* Return if it is debug insn for the consistency with optimize == 0. */
- if (insn->is_debug_insn ())
- return;
+private:
+ insn_info *m_insn;
+ bb_info *m_bb;
+ rtx m_avl;
+ rtx m_vl;
+ set_info *m_avl_def;
+ uint8_t m_sew;
+ uint8_t m_max_sew;
+ vlmul_type m_vlmul;
+ uint8_t m_ratio;
+ bool m_ta;
+ bool m_ma;
+
+ sew_lmul_demand_type m_sew_lmul_demand;
+ policy_demand_type m_policy_demand;
+ avl_demand_type m_avl_demand;
+
+ enum class state_type
+ {
+ UNINITIALIZED,
+ VALID,
+ UNKNOWN,
+ EMPTY,
+ };
+ state_type m_state;
+
+ bool m_ignore;
+ bool change_vtype_only;
+ insn_info *m_read_vl_insn;
+ bool use_by_non_rvv_insn;
- /* We set it as unknown since we don't what will happen in CALL or ASM. */
- if (insn->is_call () || insn->is_asm ())
- {
- set_unknown ();
+public:
+ vsetvl_info ()
+ : m_insn (nullptr), m_bb (nullptr), m_avl (NULL_RTX), m_vl (NULL_RTX),
+ m_avl_def (nullptr), m_sew (0), m_max_sew (0), m_vlmul (LMUL_RESERVED),
+ m_ratio (0), m_ta (false), m_ma (false),
+ m_sew_lmul_demand (sew_lmul_demand_type::sew_lmul),
+ m_policy_demand (policy_demand_type::tail_mask_policy),
+ m_avl_demand (avl_demand_type::avl), m_state (state_type::UNINITIALIZED),
+ m_ignore (false), change_vtype_only (false), m_read_vl_insn (nullptr),
+ use_by_non_rvv_insn (false)
+ {}
+
+ vsetvl_info (insn_info *insn) : vsetvl_info () { parse_insn (insn); }
+
+ vsetvl_info (rtx_insn *insn) : vsetvl_info () { parse_insn (insn); }
+
+ void set_avl (rtx avl) { m_avl = avl; }
+ void set_vl (rtx vl) { m_vl = vl; }
+ void set_avl_def (set_info *avl_def) { m_avl_def = avl_def; }
+ void set_sew (uint8_t sew) { m_sew = sew; }
+ void set_vlmul (vlmul_type vlmul) { m_vlmul = vlmul; }
+ void set_ratio (uint8_t ratio) { m_ratio = ratio; }
+ void set_ta (bool ta) { m_ta = ta; }
+ void set_ma (bool ma) { m_ma = ma; }
+ void set_ignore () { m_ignore = true; }
+ void set_bb (bb_info *bb) { m_bb = bb; }
+ void set_max_sew (uint8_t max_sew) { m_max_sew = max_sew; }
+ void set_change_vtype_only () { change_vtype_only = true; }
+ void set_read_vl_insn (insn_info *insn) { m_read_vl_insn = insn; }
+
+ rtx get_avl () const { return m_avl; }
+ rtx get_vl () const { return m_vl; }
+ set_info *get_avl_def () const { return m_avl_def; }
+ uint8_t get_sew () const { return m_sew; }
+ vlmul_type get_vlmul () const { return m_vlmul; }
+ uint8_t get_ratio () const { return m_ratio; }
+ bool get_ta () const { return m_ta; }
+ bool get_ma () const { return m_ma; }
+ insn_info *get_insn () const { return m_insn; }
+ bool ignore_p () const { return m_ignore; }
+ bb_info *get_bb () const { return m_bb; }
+ uint8_t get_max_sew () const { return m_max_sew; }
+ insn_info *get_read_vl_insn () const { return m_read_vl_insn; }
+ bool use_by_non_rvv_insn_p () const { return use_by_non_rvv_insn; }
+
+ bool has_imm_avl () const { return m_avl && CONST_INT_P (m_avl); }
+ bool has_vlmax_avl () const { return vlmax_avl_p (m_avl); }
+ bool has_reg_avl () const
+ {
+ return m_avl && REG_P (m_avl) && !has_vlmax_avl ();
+ }
+ bool has_non_zero_avl () const
+ {
+ if (has_imm_avl ())
+ return INTVAL (m_avl) > 0;
+ return has_vlmax_avl ();
+ }
+ bool has_reg_vl () const
+ {
+ gcc_assert (!m_vl || REG_P (m_vl));
+ return m_vl && REG_P (m_vl);
+ }
+ bool has_same_ratio (const vsetvl_info &other) const
+ {
+ return get_ratio () == other.get_ratio ();
+ }
+ bool is_in_origin_bb () const { return get_insn ()->bb () == get_bb (); }
+ void update_avl (const vsetvl_info &other)
+ {
+ m_avl = other.get_avl ();
+ m_vl = other.get_vl ();
+ m_avl_def = other.get_avl_def ();
+ }
+
+ bool uninit_p () const { return m_state == state_type::UNINITIALIZED; }
+ bool valid_p () const { return m_state == state_type::VALID; }
+ bool unknown_p () const { return m_state == state_type::UNKNOWN; }
+ bool empty_p () const { return m_state == state_type::EMPTY; }
+ bool change_vtype_only_p () const { return change_vtype_only; }
+
+ void set_valid () { m_state = state_type::VALID; }
+ void set_unknown () { m_state = state_type::UNKNOWN; }
+ void set_empty () { m_state = state_type::EMPTY; }
+
+ void set_sew_lmul_demand (sew_lmul_demand_type demand)
+ {
+ m_sew_lmul_demand = demand;
+ }
+ void set_policy_demand (policy_demand_type demand)
+ {
+ m_policy_demand = demand;
+ }
+ void set_avl_demand (avl_demand_type demand) { m_avl_demand = demand; }
+
+ sew_lmul_demand_type get_sew_lmul_demand () const
+ {
+ return m_sew_lmul_demand;
+ }
+ policy_demand_type get_policy_demand () const { return m_policy_demand; }
+ avl_demand_type get_avl_demand () const { return m_avl_demand; }
+
+ void normalize_demand (unsigned demand_flags)
+ {
+ switch (demand_flags
+ & (DEMAND_SEW_P | DEMAND_LMUL_P | DEMAND_RATIO_P | DEMAND_GE_SEW_P))
+ {
+ case (unsigned) sew_lmul_demand_type::sew_lmul:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_lmul;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::sew_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ge_sew;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_and_ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_and_ge_sew;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_TAIL_POLICY_P | DEMAND_MASK_POLICY_P))
+ {
+ case (unsigned) policy_demand_type::tail_mask_policy:
+ m_policy_demand = policy_demand_type::tail_mask_policy;
+ break;
+ case (unsigned) policy_demand_type::tail_policy_only:
+ m_policy_demand = policy_demand_type::tail_policy_only;
+ break;
+ case (unsigned) policy_demand_type::mask_policy_only:
+ m_policy_demand = policy_demand_type::mask_policy_only;
+ break;
+ case (unsigned) policy_demand_type::ignore_policy:
+ m_policy_demand = policy_demand_type::ignore_policy;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_AVL_P | DEMAND_NON_ZERO_AVL_P))
+ {
+ case (unsigned) avl_demand_type::avl:
+ m_avl_demand = avl_demand_type::avl;
+ break;
+ case (unsigned) avl_demand_type::non_zero_avl:
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ break;
+ case (unsigned) avl_demand_type::ignore_avl:
+ m_avl_demand = avl_demand_type::ignore_avl;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ }
+
+ void parse_insn (rtx_insn *rinsn)
+ {
+ if (!NONDEBUG_INSN_P (rinsn))
return;
- }
-
- /* If this is something that updates VL/VTYPE that we don't know about, set
- the state to unknown. */
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
- && (find_access (insn->defs (), VL_REGNUM)
- || find_access (insn->defs (), VTYPE_REGNUM)))
- {
- set_unknown ();
+ if (optimize == 0 && !has_vtype_op (rinsn))
return;
- }
-
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
- return;
-
- /* Warning: This function has to work on both the lowered (i.e. post
- emit_local_forward_vsetvls) and pre-lowering forms. The main implication
- of this is that it can't use the value of a SEW, VL, or Policy operand as
- they might be stale after lowering. */
- vl_vtype_info::operator= (get_vl_vtype_info (insn));
- m_insn = insn;
- m_state = VALID;
- if (vector_config_insn_p (insn->rtl ()))
- {
- m_demands[DEMAND_AVL] = true;
- m_demands[DEMAND_RATIO] = true;
+ gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
+ set_valid ();
+ extract_insn_cached (rinsn);
+ m_avl = ::get_avl (rinsn);
+ if (has_vlmax_avl () || vsetvl_insn_p (rinsn))
+ m_vl = ::get_vl (rinsn);
+ m_sew = ::get_sew (rinsn);
+ m_vlmul = ::get_vlmul (rinsn);
+ m_ta = tail_agnostic_p (rinsn);
+ m_ma = mask_agnostic_p (rinsn);
+ }
+
+ void parse_insn (insn_info *insn)
+ {
+ m_insn = insn;
+ m_bb = insn->bb ();
+ /* Return if it is debug insn for the consistency with optimize == 0. */
+ if (insn->is_debug_insn ())
return;
- }
-
- if (has_vl_op (insn->rtl ()))
- m_demands[DEMAND_AVL] = true;
-
- if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_RATIO] = true;
- else
- {
- /* TODO: By default, if it doesn't demand RATIO, we set it
- demand SEW && LMUL both. Some instructions may demand SEW
- only and ignore LMUL, will fix it later. */
- m_demands[DEMAND_SEW] = true;
- if (!ignore_vlmul_insn_p (insn->rtl ()))
- m_demands[DEMAND_LMUL] = true;
- }
-
- if (get_attr_ta (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_TAIL_POLICY] = true;
- if (get_attr_ma (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_MASK_POLICY] = true;
-
- if (vector_config_insn_p (insn->rtl ()))
- return;
-
- if (scalar_move_insn_p (insn->rtl ()))
- {
- if (m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
- if (m_ta)
- m_demands[DEMAND_GE_SEW] = true;
- }
-
- if (!m_avl.has_avl_reg () || vlmax_avl_p (get_avl ()) || !m_avl.get_source ())
- return;
- if (!m_avl.get_source ()->insn ()->is_real ()
- && !m_avl.get_source ()->insn ()->is_phi ())
- return;
-
- insn_info *def_insn = extract_single_source (m_avl.get_source ());
- if (!def_insn || !vsetvl_insn_p (def_insn->rtl ()))
- return;
-
- vector_insn_info new_info;
- new_info.parse_insn (def_insn);
- if (!same_vlmax_p (new_info) && !scalar_move_insn_p (insn->rtl ()))
- return;
-
- if (new_info.has_avl ())
- {
- if (new_info.has_avl_imm ())
- set_avl_info (avl_info (new_info.get_avl (), nullptr));
- else
- {
- if (vlmax_avl_p (new_info.get_avl ()))
- set_avl_info (avl_info (new_info.get_avl (), get_avl_source ()));
- else
- {
- /* Conservatively propagate non-VLMAX AVL of user vsetvl:
- 1. The user vsetvl should be same block with the rvv insn.
- 2. The user vsetvl is the only def insn of rvv insn.
- 3. The AVL is not modified between def-use chain.
- 4. The VL is only used by insn within EBB.
- */
- bool modified_p = false;
- for (insn_info *i = def_insn->next_nondebug_insn ();
- real_insn_and_same_bb_p (i, get_insn ()->bb ());
- i = i->next_nondebug_insn ())
- {
- /* Consider this following sequence:
-
- insn 1: vsetvli a5,a3,e8,mf4,ta,mu
- insn 2: vsetvli zero,a5,e32,m1,ta,ma
- ...
- vle32.v v1,0(a1)
- vsetvli a2,zero,e32,m1,ta,ma
- vadd.vv v1,v1,v1
- vsetvli zero,a5,e32,m1,ta,ma
- vse32.v v1,0(a0)
- ...
- insn 3: sub a3,a3,a5
- ...
-
- We can local AVL propagate "a3" from insn 1 to insn 2
- if no insns between insn 1 and insn 2 modify "a3 even
- though insn 3 modifies "a3".
- Otherwise, we can't perform local AVL propagation.
-
- Early break if we reach the insn 2. */
- if (!before_p (i, insn))
- break;
- if (find_access (i->defs (), REGNO (new_info.get_avl ())))
- {
- modified_p = true;
- break;
- }
- }
-
- bool has_live_out_use = false;
- for (use_info *use : m_avl.get_source ()->all_uses ())
- {
- if (use->is_live_out_use ())
- {
- has_live_out_use = true;
- break;
- }
- }
- if (!modified_p && !has_live_out_use
- && def_insn == m_avl.get_source ()->insn ()
- && m_insn->bb () == def_insn->bb ())
- set_avl_info (new_info.get_avl_info ());
- }
- }
- }
- if (scalar_move_insn_p (insn->rtl ()) && m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
-}
-
-bool
-vector_insn_info::compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
-
- for (const auto &cond : incompatible_conds)
- if (cond.dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::skip_avl_compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
- unsigned array_size = sizeof (incompatible_conds) / sizeof (demands_cond);
- /* Bypass AVL incompatible cases. */
- for (unsigned i = 1; i < array_size; i++)
- if (incompatible_conds[i].dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::compatible_avl_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other.get_avl_info ();
-}
-
-bool
-vector_insn_info::compatible_avl_p (const avl_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- gcc_assert (demand_p (DEMAND_AVL) && "Can't compare AVL undemand state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other;
-}
-
-bool
-vector_insn_info::compatible_vtype_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare VTYPE in unknown state");
- if (demand_p (DEMAND_SEW))
- {
- if (!demand_p (DEMAND_GE_SEW) && m_sew != other.get_sew ())
- return false;
- if (demand_p (DEMAND_GE_SEW) && m_sew > other.get_sew ())
- return false;
- }
- if (demand_p (DEMAND_LMUL) && m_vlmul != other.get_vlmul ())
- return false;
- if (demand_p (DEMAND_RATIO) && m_ratio != other.get_ratio ())
- return false;
- if (demand_p (DEMAND_TAIL_POLICY) && m_ta != other.get_ta ())
- return false;
- if (demand_p (DEMAND_MASK_POLICY) && m_ma != other.get_ma ())
- return false;
- return true;
-}
-
-/* Determine whether the vector instructions requirements represented by
- Require are compatible with the previous vsetvli instruction represented
- by this. INSN is the instruction whose requirements we're considering. */
-bool
-vector_insn_info::compatible_p (const vl_vtype_info &curr_info) const
-{
- gcc_assert (!uninit_p () && "Can't handle uninitialized info");
- if (empty_p ())
- return false;
-
- /* Nothing is compatible with Unknown. */
- if (unknown_p ())
- return false;
-
- /* If the instruction doesn't need an AVLReg and the SEW matches, consider
- it compatible. */
- if (!demand_p (DEMAND_AVL))
- if (m_sew == curr_info.get_sew ())
- return true;
-
- return compatible_avl_p (curr_info) && compatible_vtype_p (curr_info);
-}
-
-bool
-vector_insn_info::available_p (const vector_insn_info &other) const
-{
- return *this >= other;
-}
-
-void
-vector_insn_info::fuse_avl (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- set_insn (info1.get_insn ());
- if (info1.demand_p (DEMAND_AVL))
- {
- if (info1.demand_p (DEMAND_NONZERO_AVL))
- {
- if (info2.demand_p (DEMAND_AVL)
- && !info2.demand_p (DEMAND_NONZERO_AVL))
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_AVL, true);
- set_demand (DEMAND_NONZERO_AVL, false);
- return;
- }
- }
- set_avl_info (info1.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info1.demand_p (DEMAND_NONZERO_AVL));
- }
- else
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info2.demand_p (DEMAND_NONZERO_AVL));
- }
- set_demand (DEMAND_AVL,
- info1.demand_p (DEMAND_AVL) || info2.demand_p (DEMAND_AVL));
-}
-
-void
-vector_insn_info::fuse_sew_lmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- /* We need to fuse sew && lmul according to demand info:
-
- 1. GE_SEW.
- 2. SEW.
- 3. LMUL.
- 4. RATIO. */
- if (same_sew_lmul_demand_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, info2.demand_p (DEMAND_SEW));
- set_demand (DEMAND_LMUL, info2.demand_p (DEMAND_LMUL));
- set_demand (DEMAND_RATIO, info2.demand_p (DEMAND_RATIO));
- set_demand (DEMAND_GE_SEW, info2.demand_p (DEMAND_GE_SEW));
- set_sew (info2.get_sew ());
- set_vlmul (info2.get_vlmul ());
- set_ratio (info2.get_ratio ());
+ /* We set it as unknown since we don't what will happen in CALL or ASM. */
+ if (insn->is_call () || insn->is_asm ())
+ {
+ set_unknown ();
+ return;
+ }
+
+ /* If this is something that updates VL/VTYPE that we don't know about, set
+ the state to unknown. */
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
+ && (find_access (insn->defs (), VL_REGNUM)
+ || find_access (insn->defs (), VTYPE_REGNUM)))
+ {
+ set_unknown ();
+ return;
+ }
+
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
+ /* uninitialized */
return;
- }
- for (const auto &rule : fuse_rules)
- {
- if (rule.pair.match_cond_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info1, info2));
- set_vlmul (rule.new_vlmul (info1, info2));
- set_ratio (rule.new_ratio (info1, info2));
- return;
- }
- if (rule.pair.match_cond_p (info2.get_demands (), info1.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info2, info1));
- set_vlmul (rule.new_vlmul (info2, info1));
- set_ratio (rule.new_ratio (info2, info1));
- return;
- }
- }
- gcc_unreachable ();
-}
-void
-vector_insn_info::fuse_tail_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info1.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else if (info2.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info2.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else
- set_ta (get_default_ta ());
-}
-
-void
-vector_insn_info::fuse_mask_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info1.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else if (info2.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info2.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else
- set_ma (get_default_ma ());
-}
-
-vector_insn_info
-vector_insn_info::local_merge (const vector_insn_info &merge_info) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
- /* For local backward data flow, we always update INSN && AVL as the
- latest INSN and AVL so that we can keep track status of each INSN. */
- new_info.fuse_avl (merge_info, *this);
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-vector_insn_info
-vector_insn_info::global_merge (const vector_insn_info &merge_info,
- unsigned int bb_index) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
-
- /* For global data flow, we should keep original INSN and AVL if they
- valid since we should keep the life information of each block.
-
- For example:
- bb 0 -> bb 1.
- We should keep INSN && AVL of bb 1 since we will eventually emit
- vsetvl instruction according to INSN and AVL of bb 1. */
- new_info.fuse_avl (*this, merge_info);
- /* Recompute the AVL source whose block index is equal to BB_INDEX. */
- if (new_info.get_avl_source ()
- && new_info.get_avl_source ()->insn ()->is_phi ()
- && new_info.get_avl_source ()->bb ()->index () != bb_index)
- {
- hash_set<set_info *> sets
- = get_all_sets (new_info.get_avl_source (), true, true, true);
- new_info.set_avl_source (nullptr);
- bool can_find_set_p = false;
- set_info *first_set = nullptr;
- for (set_info *set : sets)
- {
- if (!first_set)
- first_set = set;
- if (set->bb ()->index () == bb_index)
- {
- gcc_assert (!can_find_set_p);
- new_info.set_avl_source (set);
- can_find_set_p = true;
- }
- }
- if (!can_find_set_p && sets.elements () == 1
- && first_set->insn ()->is_real ())
- new_info.set_avl_source (first_set);
- }
-
- /* Make sure VLMAX AVL always has a set_info the get VL. */
- if (vlmax_avl_p (new_info.get_avl ()))
- {
- if (this->get_avl_source ())
- new_info.set_avl_source (this->get_avl_source ());
- else
- {
- gcc_assert (merge_info.get_avl_source ());
- new_info.set_avl_source (merge_info.get_avl_source ());
- }
- }
-
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-/* Wrapper helps to return the AVL or VL operand for the
- vector_insn_info. Return AVL if the AVL is not VLMAX.
- Otherwise, return the VL operand. */
-rtx
-vector_insn_info::get_avl_or_vl_reg (void) const
-{
- gcc_assert (has_avl_reg ());
- if (!vlmax_avl_p (get_avl ()))
- return get_avl ();
-
- rtx_insn *rinsn = get_insn ()->rtl ();
- if (has_vl_op (rinsn) || vsetvl_insn_p (rinsn))
- {
- rtx vl = ::get_vl (rinsn);
- /* For VLMAX, we should make sure we get the
- REG to emit 'vsetvl VL,zero' since the 'VL'
- should be the REG according to RVV ISA. */
- if (REG_P (vl))
- return vl;
- }
-
- /* We always has avl_source if it is VLMAX AVL. */
- gcc_assert (get_avl_source ());
- return get_avl_reg_rtx ();
-}
+ set_valid ();
+
+ m_avl = ::get_avl (insn->rtl ());
+ if (m_avl)
+ {
+ if (vsetvl_insn_p (insn->rtl ()) || has_vlmax_avl ())
+ m_vl = ::get_vl (insn->rtl ());
+
+ if (has_reg_avl ())
+ m_avl_def = find_access (insn->uses (), REGNO (m_avl))->def ();
+ }
+
+ m_sew = ::get_sew (insn->rtl ());
+ m_vlmul = ::get_vlmul (insn->rtl ());
+ m_ratio = get_attr_ratio (insn->rtl ());
+ /* when get_attr_ratio is invalid, this kind of instructions
+ doesn't care about ratio. However, we still need this value
+ in demand info backward analysis. */
+ if (m_ratio == INVALID_ATTRIBUTE)
+ m_ratio = calculate_ratio (m_sew, m_vlmul);
+ m_ta = tail_agnostic_p (insn->rtl ());
+ m_ma = mask_agnostic_p (insn->rtl ());
+
+ /* If merge operand is undef value, we prefer agnostic. */
+ int merge_op_idx = get_attr_merge_op_idx (insn->rtl ());
+ if (merge_op_idx != INVALID_ATTRIBUTE
+ && satisfies_constraint_vu (recog_data.operand[merge_op_idx]))
+ {
+ m_ta = true;
+ m_ma = true;
+ }
+
+ /* Determine the demand info of the RVV insn. */
+ m_max_sew = get_max_int_sew ();
+ unsigned demand_flags = 0;
+ if (vector_config_insn_p (insn->rtl ()))
+ {
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ }
+ else
+ {
+ if (has_vl_op (insn->rtl ()))
+ {
+ if (scalar_move_insn_p (insn->rtl ()))
+ {
+ /* If the avl for vmv.s.x comes from the vsetvl instruction, we
+ don't know if the avl is non-zero, so it is set to
+ DEMAND_AVL_P for now. it may be corrected to
+ DEMAND_NON_ZERO_AVL_P later when more information is
+ available.
+ */
+ if (has_non_zero_avl ())
+ demand_flags |= demand_flags::DEMAND_NON_ZERO_AVL_P;
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
-bool
-vector_insn_info::update_fault_first_load_avl (insn_info *insn)
-{
- // Update AVL to vl-output of the fault first load.
- const insn_info *read_vl = get_forward_read_vl_insn (insn);
- if (read_vl)
- {
- rtx vl = SET_DEST (PATTERN (read_vl->rtl ()));
- def_info *def = find_access (read_vl->defs (), REGNO (vl));
- set_info *set = safe_dyn_cast<set_info *> (def);
- set_avl_info (avl_info (vl, set));
- set_insn (insn);
- return true;
- }
- return false;
-}
+ if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ else
+ {
+ if (scalar_move_insn_p (insn->rtl ()) && m_ta)
+ {
+ demand_flags |= demand_flags::DEMAND_GE_SEW_P;
+ m_max_sew = get_attr_type (insn->rtl ()) == TYPE_VFMOVFV
+ ? get_max_float_sew ()
+ : get_max_int_sew ();
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_SEW_P;
+
+ if (!ignore_vlmul_insn_p (insn->rtl ()))
+ demand_flags |= demand_flags::DEMAND_LMUL_P;
+ }
-static const char *
-vlmul_to_str (vlmul_type vlmul)
-{
- switch (vlmul)
- {
- case LMUL_1:
- return "m1";
- case LMUL_2:
- return "m2";
- case LMUL_4:
- return "m4";
- case LMUL_8:
- return "m8";
- case LMUL_RESERVED:
- return "INVALID LMUL";
- case LMUL_F8:
- return "mf8";
- case LMUL_F4:
- return "mf4";
- case LMUL_F2:
- return "mf2";
+ if (!m_ta)
+ demand_flags |= demand_flags::DEMAND_TAIL_POLICY_P;
+ if (!m_ma)
+ demand_flags |= demand_flags::DEMAND_MASK_POLICY_P;
+ }
+
+ normalize_demand (demand_flags);
+
+ /* Optimize AVL from the vsetvl instruction. */
+ insn_info *def_insn = extract_single_source (get_avl_def ());
+ if (def_insn && vsetvl_insn_p (def_insn->rtl ()))
+ {
+ vsetvl_info def_info = vsetvl_info (def_insn);
+ if ((scalar_move_insn_p (insn->rtl ())
+ || def_info.get_ratio () == get_ratio ())
+ && (def_info.has_vlmax_avl () || def_info.has_imm_avl ()))
+ {
+ update_avl (def_info);
+ if (scalar_move_insn_p (insn->rtl ()) && has_non_zero_avl ())
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ }
+ }
+
+ /* Determine if dest operand(vl) has been used by non-RVV instructions. */
+ if (has_reg_vl ())
+ {
+ const hash_set<use_info *> vl_uses
+ = get_all_real_uses (get_insn (), REGNO (get_vl ()));
+ for (use_info *use : vl_uses)
+ {
+ gcc_assert (use->insn ()->is_real ());
+ rtx_insn *rinsn = use->insn ()->rtl ();
+ if (!has_vl_op (rinsn)
+ || count_regno_occurrences (rinsn, REGNO (get_vl ())) != 1)
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ rtx avl = ::get_avl (rinsn);
+ if (!avl || REGNO (get_vl ()) != REGNO (avl))
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ }
+ }
- default:
+ /* Collect the read vl insn for the fault-only-first rvv loads. */
+ if (fault_first_load_p (insn->rtl ()))
+ {
+ for (insn_info *i = insn->next_nondebug_insn ();
+ i->bb () == insn->bb (); i = i->next_nondebug_insn ())
+ {
+ if (find_access (i->defs (), VL_REGNUM))
+ break;
+ if (i->rtl () && read_vl_insn_p (i->rtl ()))
+ {
+ m_read_vl_insn = i;
+ break;
+ }
+ }
+ }
+ }
+
+ bool operator== (const vsetvl_info &other) const
+ {
+ gcc_assert (!uninit_p () && !other.uninit_p ()
+ && "Uninitialization should not happen");
+
+ if (empty_p ())
+ return other.empty_p ();
+ if (unknown_p ())
+ return other.unknown_p ();
+
+ return get_insn () == other.get_insn () && get_bb () == other.get_bb ()
+ && get_avl () == other.get_avl () && get_vl () == other.get_vl ()
+ && get_avl_def () == other.get_avl_def ()
+ && get_sew () == other.get_sew ()
+ && get_vlmul () == other.get_vlmul () && get_ta () == other.get_ta ()
+ && get_ma () == other.get_ma ()
+ && get_avl_demand () == other.get_avl_demand ()
+ && get_sew_lmul_demand () == other.get_sew_lmul_demand ()
+ && get_policy_demand () == other.get_policy_demand ();
+ }
+
+ void dump (FILE *file, const char *indent = "") const
+ {
+ if (uninit_p ())
+ {
+ fprintf (file, "UNINITIALIZED.\n");
+ return;
+ }
+ else if (unknown_p ())
+ {
+ fprintf (file, "UNKNOWN.\n");
+ return;
+ }
+ else if (empty_p ())
+ {
+ fprintf (file, "EMPTY.\n");
+ return;
+ }
+ else if (valid_p ())
+ fprintf (file, "VALID (insn %u, bb %u)%s\n", get_insn ()->uid (),
+ get_bb ()->index (), ignore_p () ? " (ignore)" : "");
+ else
gcc_unreachable ();
- }
-}
-
-static const char *
-policy_to_str (bool agnostic_p)
-{
- return agnostic_p ? "agnostic" : "undisturbed";
-}
-void
-vector_insn_info::dump (FILE *file) const
-{
- fprintf (file, "[");
- if (uninit_p ())
- fprintf (file, "UNINITIALIZED,");
- else if (valid_p ())
- fprintf (file, "VALID,");
- else if (unknown_p ())
- fprintf (file, "UNKNOWN,");
- else if (empty_p ())
- fprintf (file, "EMPTY,");
- else
- fprintf (file, "DIRTY,");
-
- fprintf (file, "Demand field={%d(VL),", demand_p (DEMAND_AVL));
- fprintf (file, "%d(DEMAND_NONZERO_AVL),", demand_p (DEMAND_NONZERO_AVL));
- fprintf (file, "%d(SEW),", demand_p (DEMAND_SEW));
- fprintf (file, "%d(DEMAND_GE_SEW),", demand_p (DEMAND_GE_SEW));
- fprintf (file, "%d(LMUL),", demand_p (DEMAND_LMUL));
- fprintf (file, "%d(RATIO),", demand_p (DEMAND_RATIO));
- fprintf (file, "%d(TAIL_POLICY),", demand_p (DEMAND_TAIL_POLICY));
- fprintf (file, "%d(MASK_POLICY)}\n", demand_p (DEMAND_MASK_POLICY));
-
- fprintf (file, "AVL=");
- print_rtl_single (file, get_avl ());
- fprintf (file, "SEW=%d,", get_sew ());
- fprintf (file, "VLMUL=%s,", vlmul_to_str (get_vlmul ()));
- fprintf (file, "RATIO=%d,", get_ratio ());
- fprintf (file, "TAIL_POLICY=%s,", policy_to_str (get_ta ()));
- fprintf (file, "MASK_POLICY=%s", policy_to_str (get_ma ()));
- fprintf (file, "]\n");
-
- if (valid_p ())
- {
- if (get_insn ())
- {
- fprintf (file, "The real INSN=");
- print_rtl_single (file, get_insn ()->rtl ());
- }
- }
-}
+ fprintf (file, "%sDemand fields:", indent);
+ if (m_sew_lmul_demand == sew_lmul_demand_type::sew_lmul)
+ fprintf (file, " demand_sew_lmul");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_only)
+ fprintf (file, " demand_ratio_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::sew_only)
+ fprintf (file, " demand_sew_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ge_sew)
+ fprintf (file, " demand_ge_sew");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_and_ge_sew)
+ fprintf (file, " demand_ratio_and_ge_sew");
+
+ if (m_policy_demand == policy_demand_type::tail_mask_policy)
+ fprintf (file, " demand_tail_mask_policy");
+ else if (m_policy_demand == policy_demand_type::tail_policy_only)
+ fprintf (file, " demand_tail_policy_only");
+ else if (m_policy_demand == policy_demand_type::mask_policy_only)
+ fprintf (file, " demand_mask_policy_only");
+
+ if (m_avl_demand == avl_demand_type::avl)
+ fprintf (file, " demand_avl");
+ else if (m_avl_demand == avl_demand_type::non_zero_avl)
+ fprintf (file, " demand_non_zero_avl");
+ fprintf (file, "\n");
+
+ fprintf (file, "%sSEW=%d, ", indent, get_sew ());
+ fprintf (file, "VLMUL=%s, ", vlmul_to_str (get_vlmul ()));
+ fprintf (file, "RATIO=%d, ", get_ratio ());
+ fprintf (file, "MAX_SEW=%d\n", get_max_sew ());
+
+ fprintf (file, "%sTAIL_POLICY=%s, ", indent, policy_to_str (get_ta ()));
+ fprintf (file, "MASK_POLICY=%s\n", policy_to_str (get_ma ()));
+
+ fprintf (file, "%sAVL=", indent);
+ print_rtl_single (file, get_avl ());
+ fprintf (file, "%sVL=", indent);
+ print_rtl_single (file, get_vl ());
+ if (change_vtype_only_p ())
+ fprintf (file, "%schange vtype only\n", indent);
+ if (get_read_vl_insn ())
+ fprintf (file, "%sread_vl_insn: insn %u\n", indent,
+ get_read_vl_insn ()->uid ());
+ if (use_by_non_rvv_insn_p ())
+ fprintf (file, "%suse_by_non_rvv_insn=true\n", indent);
+ }
+};
vector_infos_manager::vector_infos_manager ()
{
diff --git a/gcc/config/riscv/riscv-vsetvl.h b/gcc/config/riscv/riscv-vsetvl.h
index 53549abfac5..9eab276190e 100644
--- a/gcc/config/riscv/riscv-vsetvl.h
+++ b/gcc/config/riscv/riscv-vsetvl.h
@@ -82,267 +82,6 @@ enum def_type
CLOBBER_DEF = 1 << 4
};
-/* AVL info for RVV instruction. Most RVV instructions have AVL operand in
- implicit dependency. The AVL comparison between 2 RVV instructions is
- very important since it affects our decision whether we should insert
- a vsetvl instruction in this situation. AVL operand of all RVV instructions
- can only be either a const_int value with < 32 or a reg value which can be
- define by either a real RTL instruction or a PHI instruction. So we need a
- standalone method to define AVL comparison and we can not simpily use
- operator "==" to compare 2 RTX value since it's to strict which will make
- use miss a lot of optimization opportunities. This method handle these
- following cases:
-
- - Background:
- Insert-vsetvl PASS is working after RA.
-
- - Terminology:
- - pr: Pseudo-register.
- - hr: Hardware-register.
-
- - Case 1:
-
- Before RA:
- li pr138,13
- insn1 (implicit depend on pr138).
- li pr138,14
- insn2 (implicit depend on pr139).
-
- After RA:
- li hr5,13
- insn1 (implicit depend on hr5).
- li hr5,14
- insn2 (implicit depend on hr5).
-
- Correct IR after vsetvl PASS:
- li hr5,13
- vsetvl1 zero,hr5....
- insn1 (implicit depend on hr5).
- li hr5,14
- vsetvl2 zero,hr5....
- insn2 (implicit depend on hr5).
-
- In this case, both insn1 and insn2 are using hr5 as the same AVL.
- If we use "rtx_equal_p" or "REGNO (AVL1) == REGNO (AVL)", we will end
- up with missing the vsetvl2 instruction which creates wrong result.
-
- Note: Using "==" operator to compare 2 AVL RTX strictly can fix this
- issue. However, it is a too strict comparison method since not all member
- variables in RTX data structure are not neccessary to be the same. It will
- make us miss a lot of optimization opportunities.
-
- - Case 2:
-
- After RA:
- bb 0:
- li hr5,13
- bb 1:
- li hr5,14
- bb2:
- insn1 (implicit depend on hr5).
- insn2 (implicit depend on hr5).
-
- In this case, we may end up with different AVL RTX and produce redundant
- vsetvl instruction.
-
- VALUE is the implicit dependency in each RVV instruction.
- SOURCE is the source definition information of AVL operand. */
-class avl_info
-{
-private:
- rtx m_value;
- rtl_ssa::set_info *m_source;
-
-public:
- avl_info () : m_value (NULL_RTX), m_source (nullptr) {}
- avl_info (const avl_info &);
- avl_info (rtx, rtl_ssa::set_info *);
- rtx get_value () const { return m_value; }
- rtl_ssa::set_info *get_source () const { return m_source; }
- void set_source (rtl_ssa::set_info *set) { m_source = set; }
- bool single_source_equal_p (const avl_info &) const;
- bool multiple_source_equal_p (const avl_info &) const;
- avl_info &operator= (const avl_info &);
- bool operator== (const avl_info &) const;
- bool operator!= (const avl_info &) const;
-
- bool has_avl_imm () const
- {
- return get_value () && CONST_INT_P (get_value ());
- }
- bool has_avl_reg () const { return get_value () && REG_P (get_value ()); }
- bool has_avl_no_reg () const { return !get_value (); }
- bool has_non_zero_avl () const;
- bool has_avl () const { return get_value (); }
-};
-
-/* Basic structure to save VL/VTYPE information. */
-struct vl_vtype_info
-{
-protected:
- /* AVL can be either register or const_int. */
- avl_info m_avl;
- /* Fields from VTYPE. The VTYPE checking depend on the flag
- dem_* before. */
- uint8_t m_sew;
- riscv_vector::vlmul_type m_vlmul;
- uint8_t m_ratio;
- bool m_ta;
- bool m_ma;
-
-public:
- void set_sew (uint8_t sew) { m_sew = sew; }
- void set_vlmul (riscv_vector::vlmul_type vlmul) { m_vlmul = vlmul; }
- void set_ratio (uint8_t ratio) { m_ratio = ratio; }
- void set_ta (bool ta) { m_ta = ta; }
- void set_ma (bool ma) { m_ma = ma; }
-
- vl_vtype_info ()
- : m_avl (avl_info ()), m_sew (0), m_vlmul (riscv_vector::LMUL_RESERVED),
- m_ratio (0), m_ta (0), m_ma (0)
- {}
- vl_vtype_info (const vl_vtype_info &) = default;
- vl_vtype_info &operator= (const vl_vtype_info &) = default;
- vl_vtype_info (avl_info, uint8_t, riscv_vector::vlmul_type, uint8_t, bool,
- bool);
-
- bool operator== (const vl_vtype_info &) const;
- bool operator!= (const vl_vtype_info &) const;
-
- bool has_avl_imm () const { return m_avl.has_avl_imm (); }
- bool has_avl_reg () const { return m_avl.has_avl_reg (); }
- bool has_avl_no_reg () const { return m_avl.has_avl_no_reg (); }
- bool has_non_zero_avl () const { return m_avl.has_non_zero_avl (); };
- bool has_avl () const { return m_avl.has_avl (); }
-
- rtx get_avl () const { return m_avl.get_value (); }
- const avl_info &get_avl_info () const { return m_avl; }
- rtl_ssa::set_info *get_avl_source () const { return m_avl.get_source (); }
- void set_avl_source (rtl_ssa::set_info *set) { m_avl.set_source (set); }
- void set_avl_info (const avl_info &avl) { m_avl = avl; }
- uint8_t get_sew () const { return m_sew; }
- riscv_vector::vlmul_type get_vlmul () const { return m_vlmul; }
- uint8_t get_ratio () const { return m_ratio; }
- bool get_ta () const { return m_ta; }
- bool get_ma () const { return m_ma; }
-
- bool same_avl_p (const vl_vtype_info &) const;
- bool same_vtype_p (const vl_vtype_info &) const;
- bool same_vlmax_p (const vl_vtype_info &) const;
-};
-
-class vector_insn_info : public vl_vtype_info
-{
-private:
- enum state_type
- {
- UNINITIALIZED,
- VALID,
- UNKNOWN,
- EMPTY,
-
- /* The block is polluted as containing VSETVL instruction during dem
- backward propagation to gain better LCM optimization even though
- such VSETVL instruction is not really emit yet during this time. */
- DIRTY,
- };
-
- enum state_type m_state;
-
- bool m_demands[NUM_DEMAND];
-
- /* TODO: Assume INSN1 = INSN holding of definition of AVL.
- INSN2 = INSN that is inserted a vsetvl insn before.
- We may need to add a new member to save INSN of holding AVL.
- m_insn is holding the INSN that is inserted a vsetvl insn before in
- Phase 2. Ideally, most of the time INSN1 == INSN2. However, considering
- such case:
-
- vmv.x.s (INSN2)
- vle8.v (INSN1)
-
- If these 2 instructions are compatible, we should only issue a vsetvl INSN
- (with AVL included) before vmv.x.s, but vmv.x.s is not the INSN holding the
- definition of AVL. */
- rtl_ssa::insn_info *m_insn;
-
- friend class vector_infos_manager;
-
-public:
- vector_insn_info ()
- : vl_vtype_info (), m_state (UNINITIALIZED), m_demands{false},
- m_insn (nullptr)
- {}
-
- /* Parse the instruction to get VL/VTYPE information and demanding
- * information. */
- /* This is only called by simple_vsetvl subroutine when optimize == 0.
- Since RTL_SSA can not be enabled when optimize == 0, we don't initialize
- the m_insn. */
- void parse_insn (rtx_insn *);
- /* This is only called by lazy_vsetvl subroutine when optimize > 0.
- We use RTL_SSA framework to initialize the insn_info. */
- void parse_insn (rtl_ssa::insn_info *);
-
- bool operator>= (const vector_insn_info &) const;
- bool operator== (const vector_insn_info &) const;
-
- bool uninit_p () const { return m_state == UNINITIALIZED; }
- bool valid_p () const { return m_state == VALID; }
- bool unknown_p () const { return m_state == UNKNOWN; }
- bool empty_p () const { return m_state == EMPTY; }
- bool dirty_p () const { return m_state == DIRTY; }
- bool valid_or_dirty_p () const
- {
- return m_state == VALID || m_state == DIRTY;
- }
- bool available_p (const vector_insn_info &) const;
-
- static vector_insn_info get_unknown ()
- {
- vector_insn_info info;
- info.set_unknown ();
- return info;
- }
-
- void set_valid () { m_state = VALID; }
- void set_unknown () { m_state = UNKNOWN; }
- void set_empty () { m_state = EMPTY; }
- void set_dirty () { m_state = DIRTY; }
- void set_insn (rtl_ssa::insn_info *insn) { m_insn = insn; }
-
- bool demand_p (enum demand_type type) const { return m_demands[type]; }
- void demand (enum demand_type type) { m_demands[type] = true; }
- void set_demand (enum demand_type type, bool value)
- {
- m_demands[type] = value;
- }
- void fuse_avl (const vector_insn_info &, const vector_insn_info &);
- void fuse_sew_lmul (const vector_insn_info &, const vector_insn_info &);
- void fuse_tail_policy (const vector_insn_info &, const vector_insn_info &);
- void fuse_mask_policy (const vector_insn_info &, const vector_insn_info &);
-
- bool compatible_p (const vector_insn_info &) const;
- bool skip_avl_compatible_p (const vector_insn_info &) const;
- bool compatible_avl_p (const vl_vtype_info &) const;
- bool compatible_avl_p (const avl_info &) const;
- bool compatible_vtype_p (const vl_vtype_info &) const;
- bool compatible_p (const vl_vtype_info &) const;
- vector_insn_info local_merge (const vector_insn_info &) const;
- vector_insn_info global_merge (const vector_insn_info &, unsigned int) const;
-
- rtl_ssa::insn_info *get_insn () const { return m_insn; }
- const bool *get_demands (void) const { return m_demands; }
- rtx get_avl_or_vl_reg (void) const;
- rtx get_avl_reg_rtx (void) const
- {
- return gen_rtx_REG (Pmode, get_avl_source ()->regno ());
- }
- bool update_fault_first_load_avl (rtl_ssa::insn_info *);
-
- void dump (FILE *) const;
-};
-
struct vector_block_info
{
/* The local_dem vector insn_info of the block. */
--
2.36.3
+ bool m_ignore;
Change it into m_delete;
And add comment:
/* It's set when VSETVL status from all predecessors are all available to the current block demand.
Such block VSETVL should be deleted in the final vsetvl insertion phase. */
bool ignore_p () const { return m_ignore; }-> bool delete_p () const { return m_delete; }
juzhe.zhong@rivai.ai
From: Lehua Ding
Date: 2023-10-17 19:34
To: gcc-patches
CC: juzhe.zhong; kito.cheng; rdapp.gcc; palmer; jeffreyalaw; lehua.ding
Subject: [PATCH V2 01/14] RISC-V: P1: Refactor avl_info/vl_vtype_info/vector_insn_info
This sub-patch combine avl_info/vl_vtype_info/vector_insn_info to
a single class vsetvl_info.
gcc/ChangeLog:
* config/riscv/riscv-vsetvl.cc (avl_info::avl_info): Removed.
(avl_info::single_source_equal_p): Ditto.
(avl_info::multiple_source_equal_p): Ditto.
(avl_info::operator=): Ditto.
(avl_info::operator==): Ditto.
(avl_info::operator!=): Ditto.
(avl_info::has_non_zero_avl): Ditto.
(vl_vtype_info::vl_vtype_info): Ditto.
(vl_vtype_info::operator==): Ditto.
(vl_vtype_info::operator!=): Ditto.
(vl_vtype_info::same_avl_p): Ditto.
(vl_vtype_info::same_vtype_p): Ditto.
(enum demand_flags): New enum.
(vl_vtype_info::same_vlmax_p): Removed.
(vector_insn_info::operator>=): Ditto.
(enum class): New demand types.
(vector_insn_info::operator==): Ditto.
(vector_insn_info::parse_insn): Ditto.
(class vsetvl_info): New class.
(vector_insn_info::compatible_p): Removed.
(vector_insn_info::skip_avl_compatible_p): Ditto.
(vector_insn_info::compatible_avl_p): Ditto.
(vector_insn_info::compatible_vtype_p): Ditto.
(vector_insn_info::available_p): Ditto.
(vector_insn_info::fuse_avl): Ditto.
(vector_insn_info::fuse_sew_lmul): Ditto.
(vector_insn_info::fuse_tail_policy): Ditto.
(vector_insn_info::fuse_mask_policy): Ditto.
(vector_insn_info::local_merge): Ditto.
(vector_insn_info::global_merge): Ditto.
(vector_insn_info::get_avl_or_vl_reg): Ditto.
(vector_insn_info::update_fault_first_load_avl): Ditto.
(vlmul_to_str): Ditto.
(policy_to_str): Ditto.
(vector_insn_info::dump): Ditto.
* config/riscv/riscv-vsetvl.h (class avl_info): Ditto.
(struct vl_vtype_info): Ditto.
(class vector_insn_info): Ditto.
---
gcc/config/riscv/riscv-vsetvl.cc | 1315 ++++++++++++------------------
gcc/config/riscv/riscv-vsetvl.h | 261 ------
2 files changed, 515 insertions(+), 1061 deletions(-)
diff --git a/gcc/config/riscv/riscv-vsetvl.cc b/gcc/config/riscv/riscv-vsetvl.cc
index 4b06d93e7f9..79ba8466556 100644
--- a/gcc/config/riscv/riscv-vsetvl.cc
+++ b/gcc/config/riscv/riscv-vsetvl.cc
@@ -1581,827 +1581,542 @@ vsetvl_dominated_by_p (const basic_block cfg_bb,
return true;
}
-avl_info::avl_info (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
-}
-
-avl_info::avl_info (rtx value_in, set_info *source_in)
- : m_value (value_in), m_source (source_in)
-{}
-
-bool
-avl_info::single_source_equal_p (const avl_info &other) const
-{
- set_info *set1 = m_source;
- set_info *set2 = other.get_source ();
- insn_info *insn1 = extract_single_source (set1);
- insn_info *insn2 = extract_single_source (set2);
- if (!insn1 || !insn2)
- return false;
- return source_equal_p (insn1, insn2);
-}
-
-bool
-avl_info::multiple_source_equal_p (const avl_info &other) const
-{
- /* When the def info is same in RTL_SSA namespace, it's safe
- to consider they are avl compatible. */
- if (m_source == other.get_source ())
- return true;
-
- /* We only consider handle PHI node. */
- if (!m_source->insn ()->is_phi () || !other.get_source ()->insn ()->is_phi ())
- return false;
-
- phi_info *phi1 = as_a<phi_info *> (m_source);
- phi_info *phi2 = as_a<phi_info *> (other.get_source ());
-
- if (phi1->is_degenerate () && phi2->is_degenerate ())
- {
- /* Degenerate PHI means the PHI node only have one input. */
-
- /* If both PHI nodes have the same single input in use list.
- We consider they are AVL compatible. */
- if (phi1->input_value (0) == phi2->input_value (0))
- return true;
- }
- /* TODO: We can support more optimization cases in the future. */
- return false;
-}
-
-avl_info &
-avl_info::operator= (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
- return *this;
-}
-
-bool
-avl_info::operator== (const avl_info &other) const
-{
- if (!m_value)
- return !other.get_value ();
- if (!other.get_value ())
- return false;
-
- if (GET_CODE (m_value) != GET_CODE (other.get_value ()))
- return false;
-
- /* Handle CONST_INT AVL. */
- if (CONST_INT_P (m_value))
- return INTVAL (m_value) == INTVAL (other.get_value ());
-
- /* Handle VLMAX AVL. */
- if (vlmax_avl_p (m_value))
- return vlmax_avl_p (other.get_value ());
- if (vlmax_avl_p (other.get_value ()))
- return false;
-
- /* If any source is undef value, we think they are not equal. */
- if (!m_source || !other.get_source ())
- return false;
-
- /* If both sources are single source (defined by a single real RTL)
- and their definitions are same. */
- if (single_source_equal_p (other))
- return true;
-
- return multiple_source_equal_p (other);
-}
-
-bool
-avl_info::operator!= (const avl_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-avl_info::has_non_zero_avl () const
-{
- if (has_avl_imm ())
- return INTVAL (get_value ()) > 0;
- if (has_avl_reg ())
- return vlmax_avl_p (get_value ());
- return false;
-}
-
-/* Initialize VL/VTYPE information. */
-vl_vtype_info::vl_vtype_info (avl_info avl_in, uint8_t sew_in,
- enum vlmul_type vlmul_in, uint8_t ratio_in,
- bool ta_in, bool ma_in)
- : m_avl (avl_in), m_sew (sew_in), m_vlmul (vlmul_in), m_ratio (ratio_in),
- m_ta (ta_in), m_ma (ma_in)
-{
- gcc_assert (valid_sew_p (m_sew) && "Unexpected SEW");
-}
-
-bool
-vl_vtype_info::operator== (const vl_vtype_info &other) const
-{
- return same_avl_p (other) && m_sew == other.get_sew ()
- && m_vlmul == other.get_vlmul () && m_ta == other.get_ta ()
- && m_ma == other.get_ma () && m_ratio == other.get_ratio ();
-}
-
-bool
-vl_vtype_info::operator!= (const vl_vtype_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-vl_vtype_info::same_avl_p (const vl_vtype_info &other) const
-{
- /* We need to compare both RTL and SET. If both AVL are CONST_INT.
- For example, const_int 3 and const_int 4, we need to compare
- RTL. If both AVL are REG and their REGNO are same, we need to
- compare SET. */
- return get_avl () == other.get_avl ()
- && get_avl_source () == other.get_avl_source ();
-}
-
-bool
-vl_vtype_info::same_vtype_p (const vl_vtype_info &other) const
-{
- return get_sew () == other.get_sew () && get_vlmul () == other.get_vlmul ()
- && get_ta () == other.get_ta () && get_ma () == other.get_ma ();
-}
+/* This flags indicates the minimum demand of the vl and vtype values by the
+ RVV instruction. For example, DEMAND_RATIO_P indicates that this RVV
+ instruction only needs the SEW/LMUL ratio to remain the same, and does not
+ require SEW and LMUL to be fixed.
+ Therefore, if the former RVV instruction needs DEMAND_RATIO_P and the latter
+ instruction needs DEMAND_SEW_LMUL_P and its SEW/LMUL is the same as that of
+ the former instruction, then we can make the minimu demand of the former
+ instruction strict to DEMAND_SEW_LMUL_P, and its required SEW and LMUL are
+ the SEW and LMUL of the latter instruction, and the vsetvl instruction
+ generated according to the new demand can also be used for the latter
+ instruction, so there is no need to insert a separate vsetvl instruction for
+ the latter instruction. */
+enum demand_flags : unsigned
+{
+ DEMAND_EMPTY_P = 0,
+ DEMAND_SEW_P = 1 << 0,
+ DEMAND_LMUL_P = 1 << 1,
+ DEMAND_RATIO_P = 1 << 2,
+ DEMAND_GE_SEW_P = 1 << 3,
+ DEMAND_TAIL_POLICY_P = 1 << 4,
+ DEMAND_MASK_POLICY_P = 1 << 5,
+ DEMAND_AVL_P = 1 << 6,
+ DEMAND_NON_ZERO_AVL_P = 1 << 7,
+};
-bool
-vl_vtype_info::same_vlmax_p (const vl_vtype_info &other) const
-{
- return get_ratio () == other.get_ratio ();
-}
+/* We split the demand information into three parts. They are sew and lmul
+ related (sew_lmul_demand_type), tail and mask policy related
+ (policy_demand_type) and avl related (avl_demand_type). Then we define three
+ interfaces avaiable_with, compatible_with and merge_with. avaiable_with is
+ used to determine whether the two vsetvl infos prev_info and next_info are
+ available or not. If prev_info is available for next_info, it means that the
+ RVV insn corresponding to next_info on the path from prev_info to next_info
+ can be used without inserting a separate vsetvl instruction. compatible_with
+ is used to determine whether prev_info is compatible with next_info, and if
+ so, merge_with can be used to merge the stricter demand information from
+ next_info into prev_info so that prev_info becomes available to next_info.
+ */
-/* Compare the compatibility between Dem1 and Dem2.
- If Dem1 > Dem2, Dem1 has bigger compatibility then Dem2
- meaning Dem1 is easier be compatible with others than Dem2
- or Dem2 is stricter than Dem1.
- For example, Dem1 (demand SEW + LMUL) > Dem2 (demand RATIO). */
-bool
-vector_insn_info::operator>= (const vector_insn_info &other) const
+enum class sew_lmul_demand_type : unsigned
{
- if (support_relaxed_compatible_p (*this, other))
- {
- unsigned array_size = sizeof (unavailable_conds) / sizeof (demands_cond);
- /* Bypass AVL unavailable cases. */
- for (unsigned i = 2; i < array_size; i++)
- if (unavailable_conds[i].pair.match_cond_p (this->get_demands (),
- other.get_demands ())
- && unavailable_conds[i].incompatible_p (*this, other))
- return false;
- return true;
- }
-
- if (!other.compatible_p (static_cast<const vl_vtype_info &> (*this)))
- return false;
- if (!this->compatible_p (static_cast<const vl_vtype_info &> (other)))
- return true;
-
- if (*this == other)
- return true;
-
- for (const auto &cond : unavailable_conds)
- if (cond.pair.match_cond_p (this->get_demands (), other.get_demands ())
- && cond.incompatible_p (*this, other))
- return false;
-
- return true;
-}
+ sew_lmul = demand_flags::DEMAND_SEW_P | demand_flags::DEMAND_LMUL_P,
+ ratio_only = demand_flags::DEMAND_RATIO_P,
+ sew_only = demand_flags::DEMAND_SEW_P,
+ ge_sew = demand_flags::DEMAND_GE_SEW_P,
+ ratio_and_ge_sew
+ = demand_flags::DEMAND_RATIO_P | demand_flags::DEMAND_GE_SEW_P,
+};
-bool
-vector_insn_info::operator== (const vector_insn_info &other) const
+enum class policy_demand_type : unsigned
{
- gcc_assert (!uninit_p () && !other.uninit_p ()
- && "Uninitialization should not happen");
-
- /* Empty is only equal to another Empty. */
- if (empty_p ())
- return other.empty_p ();
- if (other.empty_p ())
- return empty_p ();
-
- /* Unknown is only equal to another Unknown. */
- if (unknown_p ())
- return other.unknown_p ();
- if (other.unknown_p ())
- return unknown_p ();
-
- for (size_t i = 0; i < NUM_DEMAND; i++)
- if (m_demands[i] != other.demand_p ((enum demand_type) i))
- return false;
-
- /* We should consider different INSN demands as different
- expression. Otherwise, we will be doing incorrect vsetvl
- elimination. */
- if (m_insn != other.get_insn ())
- return false;
-
- if (!same_avl_p (other))
- return false;
-
- /* If the full VTYPE is valid, check that it is the same. */
- return same_vtype_p (other);
-}
+ tail_mask_policy
+ = demand_flags::DEMAND_TAIL_POLICY_P | demand_flags::DEMAND_MASK_POLICY_P,
+ tail_policy_only = demand_flags::DEMAND_TAIL_POLICY_P,
+ mask_policy_only = demand_flags::DEMAND_MASK_POLICY_P,
+ ignore_policy = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (rtx_insn *rinsn)
+enum class avl_demand_type : unsigned
{
- *this = vector_insn_info ();
- if (!NONDEBUG_INSN_P (rinsn))
- return;
- if (optimize == 0 && !has_vtype_op (rinsn))
- return;
- gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
- m_state = VALID;
- extract_insn_cached (rinsn);
- rtx avl = ::get_avl (rinsn);
- m_avl = avl_info (avl, nullptr);
- m_sew = ::get_sew (rinsn);
- m_vlmul = ::get_vlmul (rinsn);
- m_ta = tail_agnostic_p (rinsn);
- m_ma = mask_agnostic_p (rinsn);
-}
+ avl = demand_flags::DEMAND_AVL_P,
+ non_zero_avl = demand_flags::DEMAND_NON_ZERO_AVL_P,
+ ignore_avl = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (insn_info *insn)
+class vsetvl_info
{
- *this = vector_insn_info ();
-
- /* Return if it is debug insn for the consistency with optimize == 0. */
- if (insn->is_debug_insn ())
- return;
+private:
+ insn_info *m_insn;
+ bb_info *m_bb;
+ rtx m_avl;
+ rtx m_vl;
+ set_info *m_avl_def;
+ uint8_t m_sew;
+ uint8_t m_max_sew;
+ vlmul_type m_vlmul;
+ uint8_t m_ratio;
+ bool m_ta;
+ bool m_ma;
+
+ sew_lmul_demand_type m_sew_lmul_demand;
+ policy_demand_type m_policy_demand;
+ avl_demand_type m_avl_demand;
+
+ enum class state_type
+ {
+ UNINITIALIZED,
+ VALID,
+ UNKNOWN,
+ EMPTY,
+ };
+ state_type m_state;
+
+ bool m_ignore;
+ bool change_vtype_only;
+ insn_info *m_read_vl_insn;
+ bool use_by_non_rvv_insn;
- /* We set it as unknown since we don't what will happen in CALL or ASM. */
- if (insn->is_call () || insn->is_asm ())
- {
- set_unknown ();
+public:
+ vsetvl_info ()
+ : m_insn (nullptr), m_bb (nullptr), m_avl (NULL_RTX), m_vl (NULL_RTX),
+ m_avl_def (nullptr), m_sew (0), m_max_sew (0), m_vlmul (LMUL_RESERVED),
+ m_ratio (0), m_ta (false), m_ma (false),
+ m_sew_lmul_demand (sew_lmul_demand_type::sew_lmul),
+ m_policy_demand (policy_demand_type::tail_mask_policy),
+ m_avl_demand (avl_demand_type::avl), m_state (state_type::UNINITIALIZED),
+ m_ignore (false), change_vtype_only (false), m_read_vl_insn (nullptr),
+ use_by_non_rvv_insn (false)
+ {}
+
+ vsetvl_info (insn_info *insn) : vsetvl_info () { parse_insn (insn); }
+
+ vsetvl_info (rtx_insn *insn) : vsetvl_info () { parse_insn (insn); }
+
+ void set_avl (rtx avl) { m_avl = avl; }
+ void set_vl (rtx vl) { m_vl = vl; }
+ void set_avl_def (set_info *avl_def) { m_avl_def = avl_def; }
+ void set_sew (uint8_t sew) { m_sew = sew; }
+ void set_vlmul (vlmul_type vlmul) { m_vlmul = vlmul; }
+ void set_ratio (uint8_t ratio) { m_ratio = ratio; }
+ void set_ta (bool ta) { m_ta = ta; }
+ void set_ma (bool ma) { m_ma = ma; }
+ void set_ignore () { m_ignore = true; }
+ void set_bb (bb_info *bb) { m_bb = bb; }
+ void set_max_sew (uint8_t max_sew) { m_max_sew = max_sew; }
+ void set_change_vtype_only () { change_vtype_only = true; }
+ void set_read_vl_insn (insn_info *insn) { m_read_vl_insn = insn; }
+
+ rtx get_avl () const { return m_avl; }
+ rtx get_vl () const { return m_vl; }
+ set_info *get_avl_def () const { return m_avl_def; }
+ uint8_t get_sew () const { return m_sew; }
+ vlmul_type get_vlmul () const { return m_vlmul; }
+ uint8_t get_ratio () const { return m_ratio; }
+ bool get_ta () const { return m_ta; }
+ bool get_ma () const { return m_ma; }
+ insn_info *get_insn () const { return m_insn; }
+ bool ignore_p () const { return m_ignore; }
+ bb_info *get_bb () const { return m_bb; }
+ uint8_t get_max_sew () const { return m_max_sew; }
+ insn_info *get_read_vl_insn () const { return m_read_vl_insn; }
+ bool use_by_non_rvv_insn_p () const { return use_by_non_rvv_insn; }
+
+ bool has_imm_avl () const { return m_avl && CONST_INT_P (m_avl); }
+ bool has_vlmax_avl () const { return vlmax_avl_p (m_avl); }
+ bool has_reg_avl () const
+ {
+ return m_avl && REG_P (m_avl) && !has_vlmax_avl ();
+ }
+ bool has_non_zero_avl () const
+ {
+ if (has_imm_avl ())
+ return INTVAL (m_avl) > 0;
+ return has_vlmax_avl ();
+ }
+ bool has_reg_vl () const
+ {
+ gcc_assert (!m_vl || REG_P (m_vl));
+ return m_vl && REG_P (m_vl);
+ }
+ bool has_same_ratio (const vsetvl_info &other) const
+ {
+ return get_ratio () == other.get_ratio ();
+ }
+ bool is_in_origin_bb () const { return get_insn ()->bb () == get_bb (); }
+ void update_avl (const vsetvl_info &other)
+ {
+ m_avl = other.get_avl ();
+ m_vl = other.get_vl ();
+ m_avl_def = other.get_avl_def ();
+ }
+
+ bool uninit_p () const { return m_state == state_type::UNINITIALIZED; }
+ bool valid_p () const { return m_state == state_type::VALID; }
+ bool unknown_p () const { return m_state == state_type::UNKNOWN; }
+ bool empty_p () const { return m_state == state_type::EMPTY; }
+ bool change_vtype_only_p () const { return change_vtype_only; }
+
+ void set_valid () { m_state = state_type::VALID; }
+ void set_unknown () { m_state = state_type::UNKNOWN; }
+ void set_empty () { m_state = state_type::EMPTY; }
+
+ void set_sew_lmul_demand (sew_lmul_demand_type demand)
+ {
+ m_sew_lmul_demand = demand;
+ }
+ void set_policy_demand (policy_demand_type demand)
+ {
+ m_policy_demand = demand;
+ }
+ void set_avl_demand (avl_demand_type demand) { m_avl_demand = demand; }
+
+ sew_lmul_demand_type get_sew_lmul_demand () const
+ {
+ return m_sew_lmul_demand;
+ }
+ policy_demand_type get_policy_demand () const { return m_policy_demand; }
+ avl_demand_type get_avl_demand () const { return m_avl_demand; }
+
+ void normalize_demand (unsigned demand_flags)
+ {
+ switch (demand_flags
+ & (DEMAND_SEW_P | DEMAND_LMUL_P | DEMAND_RATIO_P | DEMAND_GE_SEW_P))
+ {
+ case (unsigned) sew_lmul_demand_type::sew_lmul:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_lmul;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::sew_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ge_sew;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_and_ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_and_ge_sew;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_TAIL_POLICY_P | DEMAND_MASK_POLICY_P))
+ {
+ case (unsigned) policy_demand_type::tail_mask_policy:
+ m_policy_demand = policy_demand_type::tail_mask_policy;
+ break;
+ case (unsigned) policy_demand_type::tail_policy_only:
+ m_policy_demand = policy_demand_type::tail_policy_only;
+ break;
+ case (unsigned) policy_demand_type::mask_policy_only:
+ m_policy_demand = policy_demand_type::mask_policy_only;
+ break;
+ case (unsigned) policy_demand_type::ignore_policy:
+ m_policy_demand = policy_demand_type::ignore_policy;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_AVL_P | DEMAND_NON_ZERO_AVL_P))
+ {
+ case (unsigned) avl_demand_type::avl:
+ m_avl_demand = avl_demand_type::avl;
+ break;
+ case (unsigned) avl_demand_type::non_zero_avl:
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ break;
+ case (unsigned) avl_demand_type::ignore_avl:
+ m_avl_demand = avl_demand_type::ignore_avl;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ }
+
+ void parse_insn (rtx_insn *rinsn)
+ {
+ if (!NONDEBUG_INSN_P (rinsn))
return;
- }
-
- /* If this is something that updates VL/VTYPE that we don't know about, set
- the state to unknown. */
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
- && (find_access (insn->defs (), VL_REGNUM)
- || find_access (insn->defs (), VTYPE_REGNUM)))
- {
- set_unknown ();
+ if (optimize == 0 && !has_vtype_op (rinsn))
return;
- }
-
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
- return;
-
- /* Warning: This function has to work on both the lowered (i.e. post
- emit_local_forward_vsetvls) and pre-lowering forms. The main implication
- of this is that it can't use the value of a SEW, VL, or Policy operand as
- they might be stale after lowering. */
- vl_vtype_info::operator= (get_vl_vtype_info (insn));
- m_insn = insn;
- m_state = VALID;
- if (vector_config_insn_p (insn->rtl ()))
- {
- m_demands[DEMAND_AVL] = true;
- m_demands[DEMAND_RATIO] = true;
+ gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
+ set_valid ();
+ extract_insn_cached (rinsn);
+ m_avl = ::get_avl (rinsn);
+ if (has_vlmax_avl () || vsetvl_insn_p (rinsn))
+ m_vl = ::get_vl (rinsn);
+ m_sew = ::get_sew (rinsn);
+ m_vlmul = ::get_vlmul (rinsn);
+ m_ta = tail_agnostic_p (rinsn);
+ m_ma = mask_agnostic_p (rinsn);
+ }
+
+ void parse_insn (insn_info *insn)
+ {
+ m_insn = insn;
+ m_bb = insn->bb ();
+ /* Return if it is debug insn for the consistency with optimize == 0. */
+ if (insn->is_debug_insn ())
return;
- }
-
- if (has_vl_op (insn->rtl ()))
- m_demands[DEMAND_AVL] = true;
-
- if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_RATIO] = true;
- else
- {
- /* TODO: By default, if it doesn't demand RATIO, we set it
- demand SEW && LMUL both. Some instructions may demand SEW
- only and ignore LMUL, will fix it later. */
- m_demands[DEMAND_SEW] = true;
- if (!ignore_vlmul_insn_p (insn->rtl ()))
- m_demands[DEMAND_LMUL] = true;
- }
-
- if (get_attr_ta (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_TAIL_POLICY] = true;
- if (get_attr_ma (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_MASK_POLICY] = true;
-
- if (vector_config_insn_p (insn->rtl ()))
- return;
-
- if (scalar_move_insn_p (insn->rtl ()))
- {
- if (m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
- if (m_ta)
- m_demands[DEMAND_GE_SEW] = true;
- }
-
- if (!m_avl.has_avl_reg () || vlmax_avl_p (get_avl ()) || !m_avl.get_source ())
- return;
- if (!m_avl.get_source ()->insn ()->is_real ()
- && !m_avl.get_source ()->insn ()->is_phi ())
- return;
-
- insn_info *def_insn = extract_single_source (m_avl.get_source ());
- if (!def_insn || !vsetvl_insn_p (def_insn->rtl ()))
- return;
-
- vector_insn_info new_info;
- new_info.parse_insn (def_insn);
- if (!same_vlmax_p (new_info) && !scalar_move_insn_p (insn->rtl ()))
- return;
-
- if (new_info.has_avl ())
- {
- if (new_info.has_avl_imm ())
- set_avl_info (avl_info (new_info.get_avl (), nullptr));
- else
- {
- if (vlmax_avl_p (new_info.get_avl ()))
- set_avl_info (avl_info (new_info.get_avl (), get_avl_source ()));
- else
- {
- /* Conservatively propagate non-VLMAX AVL of user vsetvl:
- 1. The user vsetvl should be same block with the rvv insn.
- 2. The user vsetvl is the only def insn of rvv insn.
- 3. The AVL is not modified between def-use chain.
- 4. The VL is only used by insn within EBB.
- */
- bool modified_p = false;
- for (insn_info *i = def_insn->next_nondebug_insn ();
- real_insn_and_same_bb_p (i, get_insn ()->bb ());
- i = i->next_nondebug_insn ())
- {
- /* Consider this following sequence:
-
- insn 1: vsetvli a5,a3,e8,mf4,ta,mu
- insn 2: vsetvli zero,a5,e32,m1,ta,ma
- ...
- vle32.v v1,0(a1)
- vsetvli a2,zero,e32,m1,ta,ma
- vadd.vv v1,v1,v1
- vsetvli zero,a5,e32,m1,ta,ma
- vse32.v v1,0(a0)
- ...
- insn 3: sub a3,a3,a5
- ...
-
- We can local AVL propagate "a3" from insn 1 to insn 2
- if no insns between insn 1 and insn 2 modify "a3 even
- though insn 3 modifies "a3".
- Otherwise, we can't perform local AVL propagation.
-
- Early break if we reach the insn 2. */
- if (!before_p (i, insn))
- break;
- if (find_access (i->defs (), REGNO (new_info.get_avl ())))
- {
- modified_p = true;
- break;
- }
- }
-
- bool has_live_out_use = false;
- for (use_info *use : m_avl.get_source ()->all_uses ())
- {
- if (use->is_live_out_use ())
- {
- has_live_out_use = true;
- break;
- }
- }
- if (!modified_p && !has_live_out_use
- && def_insn == m_avl.get_source ()->insn ()
- && m_insn->bb () == def_insn->bb ())
- set_avl_info (new_info.get_avl_info ());
- }
- }
- }
- if (scalar_move_insn_p (insn->rtl ()) && m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
-}
-
-bool
-vector_insn_info::compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
-
- for (const auto &cond : incompatible_conds)
- if (cond.dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::skip_avl_compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
- unsigned array_size = sizeof (incompatible_conds) / sizeof (demands_cond);
- /* Bypass AVL incompatible cases. */
- for (unsigned i = 1; i < array_size; i++)
- if (incompatible_conds[i].dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::compatible_avl_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other.get_avl_info ();
-}
-
-bool
-vector_insn_info::compatible_avl_p (const avl_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- gcc_assert (demand_p (DEMAND_AVL) && "Can't compare AVL undemand state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other;
-}
-
-bool
-vector_insn_info::compatible_vtype_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare VTYPE in unknown state");
- if (demand_p (DEMAND_SEW))
- {
- if (!demand_p (DEMAND_GE_SEW) && m_sew != other.get_sew ())
- return false;
- if (demand_p (DEMAND_GE_SEW) && m_sew > other.get_sew ())
- return false;
- }
- if (demand_p (DEMAND_LMUL) && m_vlmul != other.get_vlmul ())
- return false;
- if (demand_p (DEMAND_RATIO) && m_ratio != other.get_ratio ())
- return false;
- if (demand_p (DEMAND_TAIL_POLICY) && m_ta != other.get_ta ())
- return false;
- if (demand_p (DEMAND_MASK_POLICY) && m_ma != other.get_ma ())
- return false;
- return true;
-}
-
-/* Determine whether the vector instructions requirements represented by
- Require are compatible with the previous vsetvli instruction represented
- by this. INSN is the instruction whose requirements we're considering. */
-bool
-vector_insn_info::compatible_p (const vl_vtype_info &curr_info) const
-{
- gcc_assert (!uninit_p () && "Can't handle uninitialized info");
- if (empty_p ())
- return false;
-
- /* Nothing is compatible with Unknown. */
- if (unknown_p ())
- return false;
-
- /* If the instruction doesn't need an AVLReg and the SEW matches, consider
- it compatible. */
- if (!demand_p (DEMAND_AVL))
- if (m_sew == curr_info.get_sew ())
- return true;
-
- return compatible_avl_p (curr_info) && compatible_vtype_p (curr_info);
-}
-
-bool
-vector_insn_info::available_p (const vector_insn_info &other) const
-{
- return *this >= other;
-}
-
-void
-vector_insn_info::fuse_avl (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- set_insn (info1.get_insn ());
- if (info1.demand_p (DEMAND_AVL))
- {
- if (info1.demand_p (DEMAND_NONZERO_AVL))
- {
- if (info2.demand_p (DEMAND_AVL)
- && !info2.demand_p (DEMAND_NONZERO_AVL))
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_AVL, true);
- set_demand (DEMAND_NONZERO_AVL, false);
- return;
- }
- }
- set_avl_info (info1.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info1.demand_p (DEMAND_NONZERO_AVL));
- }
- else
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info2.demand_p (DEMAND_NONZERO_AVL));
- }
- set_demand (DEMAND_AVL,
- info1.demand_p (DEMAND_AVL) || info2.demand_p (DEMAND_AVL));
-}
-
-void
-vector_insn_info::fuse_sew_lmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- /* We need to fuse sew && lmul according to demand info:
-
- 1. GE_SEW.
- 2. SEW.
- 3. LMUL.
- 4. RATIO. */
- if (same_sew_lmul_demand_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, info2.demand_p (DEMAND_SEW));
- set_demand (DEMAND_LMUL, info2.demand_p (DEMAND_LMUL));
- set_demand (DEMAND_RATIO, info2.demand_p (DEMAND_RATIO));
- set_demand (DEMAND_GE_SEW, info2.demand_p (DEMAND_GE_SEW));
- set_sew (info2.get_sew ());
- set_vlmul (info2.get_vlmul ());
- set_ratio (info2.get_ratio ());
+ /* We set it as unknown since we don't what will happen in CALL or ASM. */
+ if (insn->is_call () || insn->is_asm ())
+ {
+ set_unknown ();
+ return;
+ }
+
+ /* If this is something that updates VL/VTYPE that we don't know about, set
+ the state to unknown. */
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
+ && (find_access (insn->defs (), VL_REGNUM)
+ || find_access (insn->defs (), VTYPE_REGNUM)))
+ {
+ set_unknown ();
+ return;
+ }
+
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
+ /* uninitialized */
return;
- }
- for (const auto &rule : fuse_rules)
- {
- if (rule.pair.match_cond_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info1, info2));
- set_vlmul (rule.new_vlmul (info1, info2));
- set_ratio (rule.new_ratio (info1, info2));
- return;
- }
- if (rule.pair.match_cond_p (info2.get_demands (), info1.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info2, info1));
- set_vlmul (rule.new_vlmul (info2, info1));
- set_ratio (rule.new_ratio (info2, info1));
- return;
- }
- }
- gcc_unreachable ();
-}
-void
-vector_insn_info::fuse_tail_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info1.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else if (info2.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info2.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else
- set_ta (get_default_ta ());
-}
-
-void
-vector_insn_info::fuse_mask_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info1.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else if (info2.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info2.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else
- set_ma (get_default_ma ());
-}
-
-vector_insn_info
-vector_insn_info::local_merge (const vector_insn_info &merge_info) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
- /* For local backward data flow, we always update INSN && AVL as the
- latest INSN and AVL so that we can keep track status of each INSN. */
- new_info.fuse_avl (merge_info, *this);
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-vector_insn_info
-vector_insn_info::global_merge (const vector_insn_info &merge_info,
- unsigned int bb_index) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
-
- /* For global data flow, we should keep original INSN and AVL if they
- valid since we should keep the life information of each block.
-
- For example:
- bb 0 -> bb 1.
- We should keep INSN && AVL of bb 1 since we will eventually emit
- vsetvl instruction according to INSN and AVL of bb 1. */
- new_info.fuse_avl (*this, merge_info);
- /* Recompute the AVL source whose block index is equal to BB_INDEX. */
- if (new_info.get_avl_source ()
- && new_info.get_avl_source ()->insn ()->is_phi ()
- && new_info.get_avl_source ()->bb ()->index () != bb_index)
- {
- hash_set<set_info *> sets
- = get_all_sets (new_info.get_avl_source (), true, true, true);
- new_info.set_avl_source (nullptr);
- bool can_find_set_p = false;
- set_info *first_set = nullptr;
- for (set_info *set : sets)
- {
- if (!first_set)
- first_set = set;
- if (set->bb ()->index () == bb_index)
- {
- gcc_assert (!can_find_set_p);
- new_info.set_avl_source (set);
- can_find_set_p = true;
- }
- }
- if (!can_find_set_p && sets.elements () == 1
- && first_set->insn ()->is_real ())
- new_info.set_avl_source (first_set);
- }
-
- /* Make sure VLMAX AVL always has a set_info the get VL. */
- if (vlmax_avl_p (new_info.get_avl ()))
- {
- if (this->get_avl_source ())
- new_info.set_avl_source (this->get_avl_source ());
- else
- {
- gcc_assert (merge_info.get_avl_source ());
- new_info.set_avl_source (merge_info.get_avl_source ());
- }
- }
-
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-/* Wrapper helps to return the AVL or VL operand for the
- vector_insn_info. Return AVL if the AVL is not VLMAX.
- Otherwise, return the VL operand. */
-rtx
-vector_insn_info::get_avl_or_vl_reg (void) const
-{
- gcc_assert (has_avl_reg ());
- if (!vlmax_avl_p (get_avl ()))
- return get_avl ();
-
- rtx_insn *rinsn = get_insn ()->rtl ();
- if (has_vl_op (rinsn) || vsetvl_insn_p (rinsn))
- {
- rtx vl = ::get_vl (rinsn);
- /* For VLMAX, we should make sure we get the
- REG to emit 'vsetvl VL,zero' since the 'VL'
- should be the REG according to RVV ISA. */
- if (REG_P (vl))
- return vl;
- }
-
- /* We always has avl_source if it is VLMAX AVL. */
- gcc_assert (get_avl_source ());
- return get_avl_reg_rtx ();
-}
+ set_valid ();
+
+ m_avl = ::get_avl (insn->rtl ());
+ if (m_avl)
+ {
+ if (vsetvl_insn_p (insn->rtl ()) || has_vlmax_avl ())
+ m_vl = ::get_vl (insn->rtl ());
+
+ if (has_reg_avl ())
+ m_avl_def = find_access (insn->uses (), REGNO (m_avl))->def ();
+ }
+
+ m_sew = ::get_sew (insn->rtl ());
+ m_vlmul = ::get_vlmul (insn->rtl ());
+ m_ratio = get_attr_ratio (insn->rtl ());
+ /* when get_attr_ratio is invalid, this kind of instructions
+ doesn't care about ratio. However, we still need this value
+ in demand info backward analysis. */
+ if (m_ratio == INVALID_ATTRIBUTE)
+ m_ratio = calculate_ratio (m_sew, m_vlmul);
+ m_ta = tail_agnostic_p (insn->rtl ());
+ m_ma = mask_agnostic_p (insn->rtl ());
+
+ /* If merge operand is undef value, we prefer agnostic. */
+ int merge_op_idx = get_attr_merge_op_idx (insn->rtl ());
+ if (merge_op_idx != INVALID_ATTRIBUTE
+ && satisfies_constraint_vu (recog_data.operand[merge_op_idx]))
+ {
+ m_ta = true;
+ m_ma = true;
+ }
+
+ /* Determine the demand info of the RVV insn. */
+ m_max_sew = get_max_int_sew ();
+ unsigned demand_flags = 0;
+ if (vector_config_insn_p (insn->rtl ()))
+ {
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ }
+ else
+ {
+ if (has_vl_op (insn->rtl ()))
+ {
+ if (scalar_move_insn_p (insn->rtl ()))
+ {
+ /* If the avl for vmv.s.x comes from the vsetvl instruction, we
+ don't know if the avl is non-zero, so it is set to
+ DEMAND_AVL_P for now. it may be corrected to
+ DEMAND_NON_ZERO_AVL_P later when more information is
+ available.
+ */
+ if (has_non_zero_avl ())
+ demand_flags |= demand_flags::DEMAND_NON_ZERO_AVL_P;
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
-bool
-vector_insn_info::update_fault_first_load_avl (insn_info *insn)
-{
- // Update AVL to vl-output of the fault first load.
- const insn_info *read_vl = get_forward_read_vl_insn (insn);
- if (read_vl)
- {
- rtx vl = SET_DEST (PATTERN (read_vl->rtl ()));
- def_info *def = find_access (read_vl->defs (), REGNO (vl));
- set_info *set = safe_dyn_cast<set_info *> (def);
- set_avl_info (avl_info (vl, set));
- set_insn (insn);
- return true;
- }
- return false;
-}
+ if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ else
+ {
+ if (scalar_move_insn_p (insn->rtl ()) && m_ta)
+ {
+ demand_flags |= demand_flags::DEMAND_GE_SEW_P;
+ m_max_sew = get_attr_type (insn->rtl ()) == TYPE_VFMOVFV
+ ? get_max_float_sew ()
+ : get_max_int_sew ();
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_SEW_P;
+
+ if (!ignore_vlmul_insn_p (insn->rtl ()))
+ demand_flags |= demand_flags::DEMAND_LMUL_P;
+ }
-static const char *
-vlmul_to_str (vlmul_type vlmul)
-{
- switch (vlmul)
- {
- case LMUL_1:
- return "m1";
- case LMUL_2:
- return "m2";
- case LMUL_4:
- return "m4";
- case LMUL_8:
- return "m8";
- case LMUL_RESERVED:
- return "INVALID LMUL";
- case LMUL_F8:
- return "mf8";
- case LMUL_F4:
- return "mf4";
- case LMUL_F2:
- return "mf2";
+ if (!m_ta)
+ demand_flags |= demand_flags::DEMAND_TAIL_POLICY_P;
+ if (!m_ma)
+ demand_flags |= demand_flags::DEMAND_MASK_POLICY_P;
+ }
+
+ normalize_demand (demand_flags);
+
+ /* Optimize AVL from the vsetvl instruction. */
+ insn_info *def_insn = extract_single_source (get_avl_def ());
+ if (def_insn && vsetvl_insn_p (def_insn->rtl ()))
+ {
+ vsetvl_info def_info = vsetvl_info (def_insn);
+ if ((scalar_move_insn_p (insn->rtl ())
+ || def_info.get_ratio () == get_ratio ())
+ && (def_info.has_vlmax_avl () || def_info.has_imm_avl ()))
+ {
+ update_avl (def_info);
+ if (scalar_move_insn_p (insn->rtl ()) && has_non_zero_avl ())
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ }
+ }
+
+ /* Determine if dest operand(vl) has been used by non-RVV instructions. */
+ if (has_reg_vl ())
+ {
+ const hash_set<use_info *> vl_uses
+ = get_all_real_uses (get_insn (), REGNO (get_vl ()));
+ for (use_info *use : vl_uses)
+ {
+ gcc_assert (use->insn ()->is_real ());
+ rtx_insn *rinsn = use->insn ()->rtl ();
+ if (!has_vl_op (rinsn)
+ || count_regno_occurrences (rinsn, REGNO (get_vl ())) != 1)
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ rtx avl = ::get_avl (rinsn);
+ if (!avl || REGNO (get_vl ()) != REGNO (avl))
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ }
+ }
- default:
+ /* Collect the read vl insn for the fault-only-first rvv loads. */
+ if (fault_first_load_p (insn->rtl ()))
+ {
+ for (insn_info *i = insn->next_nondebug_insn ();
+ i->bb () == insn->bb (); i = i->next_nondebug_insn ())
+ {
+ if (find_access (i->defs (), VL_REGNUM))
+ break;
+ if (i->rtl () && read_vl_insn_p (i->rtl ()))
+ {
+ m_read_vl_insn = i;
+ break;
+ }
+ }
+ }
+ }
+
+ bool operator== (const vsetvl_info &other) const
+ {
+ gcc_assert (!uninit_p () && !other.uninit_p ()
+ && "Uninitialization should not happen");
+
+ if (empty_p ())
+ return other.empty_p ();
+ if (unknown_p ())
+ return other.unknown_p ();
+
+ return get_insn () == other.get_insn () && get_bb () == other.get_bb ()
+ && get_avl () == other.get_avl () && get_vl () == other.get_vl ()
+ && get_avl_def () == other.get_avl_def ()
+ && get_sew () == other.get_sew ()
+ && get_vlmul () == other.get_vlmul () && get_ta () == other.get_ta ()
+ && get_ma () == other.get_ma ()
+ && get_avl_demand () == other.get_avl_demand ()
+ && get_sew_lmul_demand () == other.get_sew_lmul_demand ()
+ && get_policy_demand () == other.get_policy_demand ();
+ }
+
+ void dump (FILE *file, const char *indent = "") const
+ {
+ if (uninit_p ())
+ {
+ fprintf (file, "UNINITIALIZED.\n");
+ return;
+ }
+ else if (unknown_p ())
+ {
+ fprintf (file, "UNKNOWN.\n");
+ return;
+ }
+ else if (empty_p ())
+ {
+ fprintf (file, "EMPTY.\n");
+ return;
+ }
+ else if (valid_p ())
+ fprintf (file, "VALID (insn %u, bb %u)%s\n", get_insn ()->uid (),
+ get_bb ()->index (), ignore_p () ? " (ignore)" : "");
+ else
gcc_unreachable ();
- }
-}
-
-static const char *
-policy_to_str (bool agnostic_p)
-{
- return agnostic_p ? "agnostic" : "undisturbed";
-}
-void
-vector_insn_info::dump (FILE *file) const
-{
- fprintf (file, "[");
- if (uninit_p ())
- fprintf (file, "UNINITIALIZED,");
- else if (valid_p ())
- fprintf (file, "VALID,");
- else if (unknown_p ())
- fprintf (file, "UNKNOWN,");
- else if (empty_p ())
- fprintf (file, "EMPTY,");
- else
- fprintf (file, "DIRTY,");
-
- fprintf (file, "Demand field={%d(VL),", demand_p (DEMAND_AVL));
- fprintf (file, "%d(DEMAND_NONZERO_AVL),", demand_p (DEMAND_NONZERO_AVL));
- fprintf (file, "%d(SEW),", demand_p (DEMAND_SEW));
- fprintf (file, "%d(DEMAND_GE_SEW),", demand_p (DEMAND_GE_SEW));
- fprintf (file, "%d(LMUL),", demand_p (DEMAND_LMUL));
- fprintf (file, "%d(RATIO),", demand_p (DEMAND_RATIO));
- fprintf (file, "%d(TAIL_POLICY),", demand_p (DEMAND_TAIL_POLICY));
- fprintf (file, "%d(MASK_POLICY)}\n", demand_p (DEMAND_MASK_POLICY));
-
- fprintf (file, "AVL=");
- print_rtl_single (file, get_avl ());
- fprintf (file, "SEW=%d,", get_sew ());
- fprintf (file, "VLMUL=%s,", vlmul_to_str (get_vlmul ()));
- fprintf (file, "RATIO=%d,", get_ratio ());
- fprintf (file, "TAIL_POLICY=%s,", policy_to_str (get_ta ()));
- fprintf (file, "MASK_POLICY=%s", policy_to_str (get_ma ()));
- fprintf (file, "]\n");
-
- if (valid_p ())
- {
- if (get_insn ())
- {
- fprintf (file, "The real INSN=");
- print_rtl_single (file, get_insn ()->rtl ());
- }
- }
-}
+ fprintf (file, "%sDemand fields:", indent);
+ if (m_sew_lmul_demand == sew_lmul_demand_type::sew_lmul)
+ fprintf (file, " demand_sew_lmul");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_only)
+ fprintf (file, " demand_ratio_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::sew_only)
+ fprintf (file, " demand_sew_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ge_sew)
+ fprintf (file, " demand_ge_sew");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_and_ge_sew)
+ fprintf (file, " demand_ratio_and_ge_sew");
+
+ if (m_policy_demand == policy_demand_type::tail_mask_policy)
+ fprintf (file, " demand_tail_mask_policy");
+ else if (m_policy_demand == policy_demand_type::tail_policy_only)
+ fprintf (file, " demand_tail_policy_only");
+ else if (m_policy_demand == policy_demand_type::mask_policy_only)
+ fprintf (file, " demand_mask_policy_only");
+
+ if (m_avl_demand == avl_demand_type::avl)
+ fprintf (file, " demand_avl");
+ else if (m_avl_demand == avl_demand_type::non_zero_avl)
+ fprintf (file, " demand_non_zero_avl");
+ fprintf (file, "\n");
+
+ fprintf (file, "%sSEW=%d, ", indent, get_sew ());
+ fprintf (file, "VLMUL=%s, ", vlmul_to_str (get_vlmul ()));
+ fprintf (file, "RATIO=%d, ", get_ratio ());
+ fprintf (file, "MAX_SEW=%d\n", get_max_sew ());
+
+ fprintf (file, "%sTAIL_POLICY=%s, ", indent, policy_to_str (get_ta ()));
+ fprintf (file, "MASK_POLICY=%s\n", policy_to_str (get_ma ()));
+
+ fprintf (file, "%sAVL=", indent);
+ print_rtl_single (file, get_avl ());
+ fprintf (file, "%sVL=", indent);
+ print_rtl_single (file, get_vl ());
+ if (change_vtype_only_p ())
+ fprintf (file, "%schange vtype only\n", indent);
+ if (get_read_vl_insn ())
+ fprintf (file, "%sread_vl_insn: insn %u\n", indent,
+ get_read_vl_insn ()->uid ());
+ if (use_by_non_rvv_insn_p ())
+ fprintf (file, "%suse_by_non_rvv_insn=true\n", indent);
+ }
+};
vector_infos_manager::vector_infos_manager ()
{
diff --git a/gcc/config/riscv/riscv-vsetvl.h b/gcc/config/riscv/riscv-vsetvl.h
index 53549abfac5..9eab276190e 100644
--- a/gcc/config/riscv/riscv-vsetvl.h
+++ b/gcc/config/riscv/riscv-vsetvl.h
@@ -82,267 +82,6 @@ enum def_type
CLOBBER_DEF = 1 << 4
};
-/* AVL info for RVV instruction. Most RVV instructions have AVL operand in
- implicit dependency. The AVL comparison between 2 RVV instructions is
- very important since it affects our decision whether we should insert
- a vsetvl instruction in this situation. AVL operand of all RVV instructions
- can only be either a const_int value with < 32 or a reg value which can be
- define by either a real RTL instruction or a PHI instruction. So we need a
- standalone method to define AVL comparison and we can not simpily use
- operator "==" to compare 2 RTX value since it's to strict which will make
- use miss a lot of optimization opportunities. This method handle these
- following cases:
-
- - Background:
- Insert-vsetvl PASS is working after RA.
-
- - Terminology:
- - pr: Pseudo-register.
- - hr: Hardware-register.
-
- - Case 1:
-
- Before RA:
- li pr138,13
- insn1 (implicit depend on pr138).
- li pr138,14
- insn2 (implicit depend on pr139).
-
- After RA:
- li hr5,13
- insn1 (implicit depend on hr5).
- li hr5,14
- insn2 (implicit depend on hr5).
-
- Correct IR after vsetvl PASS:
- li hr5,13
- vsetvl1 zero,hr5....
- insn1 (implicit depend on hr5).
- li hr5,14
- vsetvl2 zero,hr5....
- insn2 (implicit depend on hr5).
-
- In this case, both insn1 and insn2 are using hr5 as the same AVL.
- If we use "rtx_equal_p" or "REGNO (AVL1) == REGNO (AVL)", we will end
- up with missing the vsetvl2 instruction which creates wrong result.
-
- Note: Using "==" operator to compare 2 AVL RTX strictly can fix this
- issue. However, it is a too strict comparison method since not all member
- variables in RTX data structure are not neccessary to be the same. It will
- make us miss a lot of optimization opportunities.
-
- - Case 2:
-
- After RA:
- bb 0:
- li hr5,13
- bb 1:
- li hr5,14
- bb2:
- insn1 (implicit depend on hr5).
- insn2 (implicit depend on hr5).
-
- In this case, we may end up with different AVL RTX and produce redundant
- vsetvl instruction.
-
- VALUE is the implicit dependency in each RVV instruction.
- SOURCE is the source definition information of AVL operand. */
-class avl_info
-{
-private:
- rtx m_value;
- rtl_ssa::set_info *m_source;
-
-public:
- avl_info () : m_value (NULL_RTX), m_source (nullptr) {}
- avl_info (const avl_info &);
- avl_info (rtx, rtl_ssa::set_info *);
- rtx get_value () const { return m_value; }
- rtl_ssa::set_info *get_source () const { return m_source; }
- void set_source (rtl_ssa::set_info *set) { m_source = set; }
- bool single_source_equal_p (const avl_info &) const;
- bool multiple_source_equal_p (const avl_info &) const;
- avl_info &operator= (const avl_info &);
- bool operator== (const avl_info &) const;
- bool operator!= (const avl_info &) const;
-
- bool has_avl_imm () const
- {
- return get_value () && CONST_INT_P (get_value ());
- }
- bool has_avl_reg () const { return get_value () && REG_P (get_value ()); }
- bool has_avl_no_reg () const { return !get_value (); }
- bool has_non_zero_avl () const;
- bool has_avl () const { return get_value (); }
-};
-
-/* Basic structure to save VL/VTYPE information. */
-struct vl_vtype_info
-{
-protected:
- /* AVL can be either register or const_int. */
- avl_info m_avl;
- /* Fields from VTYPE. The VTYPE checking depend on the flag
- dem_* before. */
- uint8_t m_sew;
- riscv_vector::vlmul_type m_vlmul;
- uint8_t m_ratio;
- bool m_ta;
- bool m_ma;
-
-public:
- void set_sew (uint8_t sew) { m_sew = sew; }
- void set_vlmul (riscv_vector::vlmul_type vlmul) { m_vlmul = vlmul; }
- void set_ratio (uint8_t ratio) { m_ratio = ratio; }
- void set_ta (bool ta) { m_ta = ta; }
- void set_ma (bool ma) { m_ma = ma; }
-
- vl_vtype_info ()
- : m_avl (avl_info ()), m_sew (0), m_vlmul (riscv_vector::LMUL_RESERVED),
- m_ratio (0), m_ta (0), m_ma (0)
- {}
- vl_vtype_info (const vl_vtype_info &) = default;
- vl_vtype_info &operator= (const vl_vtype_info &) = default;
- vl_vtype_info (avl_info, uint8_t, riscv_vector::vlmul_type, uint8_t, bool,
- bool);
-
- bool operator== (const vl_vtype_info &) const;
- bool operator!= (const vl_vtype_info &) const;
-
- bool has_avl_imm () const { return m_avl.has_avl_imm (); }
- bool has_avl_reg () const { return m_avl.has_avl_reg (); }
- bool has_avl_no_reg () const { return m_avl.has_avl_no_reg (); }
- bool has_non_zero_avl () const { return m_avl.has_non_zero_avl (); };
- bool has_avl () const { return m_avl.has_avl (); }
-
- rtx get_avl () const { return m_avl.get_value (); }
- const avl_info &get_avl_info () const { return m_avl; }
- rtl_ssa::set_info *get_avl_source () const { return m_avl.get_source (); }
- void set_avl_source (rtl_ssa::set_info *set) { m_avl.set_source (set); }
- void set_avl_info (const avl_info &avl) { m_avl = avl; }
- uint8_t get_sew () const { return m_sew; }
- riscv_vector::vlmul_type get_vlmul () const { return m_vlmul; }
- uint8_t get_ratio () const { return m_ratio; }
- bool get_ta () const { return m_ta; }
- bool get_ma () const { return m_ma; }
-
- bool same_avl_p (const vl_vtype_info &) const;
- bool same_vtype_p (const vl_vtype_info &) const;
- bool same_vlmax_p (const vl_vtype_info &) const;
-};
-
-class vector_insn_info : public vl_vtype_info
-{
-private:
- enum state_type
- {
- UNINITIALIZED,
- VALID,
- UNKNOWN,
- EMPTY,
-
- /* The block is polluted as containing VSETVL instruction during dem
- backward propagation to gain better LCM optimization even though
- such VSETVL instruction is not really emit yet during this time. */
- DIRTY,
- };
-
- enum state_type m_state;
-
- bool m_demands[NUM_DEMAND];
-
- /* TODO: Assume INSN1 = INSN holding of definition of AVL.
- INSN2 = INSN that is inserted a vsetvl insn before.
- We may need to add a new member to save INSN of holding AVL.
- m_insn is holding the INSN that is inserted a vsetvl insn before in
- Phase 2. Ideally, most of the time INSN1 == INSN2. However, considering
- such case:
-
- vmv.x.s (INSN2)
- vle8.v (INSN1)
-
- If these 2 instructions are compatible, we should only issue a vsetvl INSN
- (with AVL included) before vmv.x.s, but vmv.x.s is not the INSN holding the
- definition of AVL. */
- rtl_ssa::insn_info *m_insn;
-
- friend class vector_infos_manager;
-
-public:
- vector_insn_info ()
- : vl_vtype_info (), m_state (UNINITIALIZED), m_demands{false},
- m_insn (nullptr)
- {}
-
- /* Parse the instruction to get VL/VTYPE information and demanding
- * information. */
- /* This is only called by simple_vsetvl subroutine when optimize == 0.
- Since RTL_SSA can not be enabled when optimize == 0, we don't initialize
- the m_insn. */
- void parse_insn (rtx_insn *);
- /* This is only called by lazy_vsetvl subroutine when optimize > 0.
- We use RTL_SSA framework to initialize the insn_info. */
- void parse_insn (rtl_ssa::insn_info *);
-
- bool operator>= (const vector_insn_info &) const;
- bool operator== (const vector_insn_info &) const;
-
- bool uninit_p () const { return m_state == UNINITIALIZED; }
- bool valid_p () const { return m_state == VALID; }
- bool unknown_p () const { return m_state == UNKNOWN; }
- bool empty_p () const { return m_state == EMPTY; }
- bool dirty_p () const { return m_state == DIRTY; }
- bool valid_or_dirty_p () const
- {
- return m_state == VALID || m_state == DIRTY;
- }
- bool available_p (const vector_insn_info &) const;
-
- static vector_insn_info get_unknown ()
- {
- vector_insn_info info;
- info.set_unknown ();
- return info;
- }
-
- void set_valid () { m_state = VALID; }
- void set_unknown () { m_state = UNKNOWN; }
- void set_empty () { m_state = EMPTY; }
- void set_dirty () { m_state = DIRTY; }
- void set_insn (rtl_ssa::insn_info *insn) { m_insn = insn; }
-
- bool demand_p (enum demand_type type) const { return m_demands[type]; }
- void demand (enum demand_type type) { m_demands[type] = true; }
- void set_demand (enum demand_type type, bool value)
- {
- m_demands[type] = value;
- }
- void fuse_avl (const vector_insn_info &, const vector_insn_info &);
- void fuse_sew_lmul (const vector_insn_info &, const vector_insn_info &);
- void fuse_tail_policy (const vector_insn_info &, const vector_insn_info &);
- void fuse_mask_policy (const vector_insn_info &, const vector_insn_info &);
-
- bool compatible_p (const vector_insn_info &) const;
- bool skip_avl_compatible_p (const vector_insn_info &) const;
- bool compatible_avl_p (const vl_vtype_info &) const;
- bool compatible_avl_p (const avl_info &) const;
- bool compatible_vtype_p (const vl_vtype_info &) const;
- bool compatible_p (const vl_vtype_info &) const;
- vector_insn_info local_merge (const vector_insn_info &) const;
- vector_insn_info global_merge (const vector_insn_info &, unsigned int) const;
-
- rtl_ssa::insn_info *get_insn () const { return m_insn; }
- const bool *get_demands (void) const { return m_demands; }
- rtx get_avl_or_vl_reg (void) const;
- rtx get_avl_reg_rtx (void) const
- {
- return gen_rtx_REG (Pmode, get_avl_source ()->regno ());
- }
- bool update_fault_first_load_avl (rtl_ssa::insn_info *);
-
- void dump (FILE *) const;
-};
-
struct vector_block_info
{
/* The local_dem vector insn_info of the block. */
--
2.36.3
@@ -1581,827 +1581,542 @@ vsetvl_dominated_by_p (const basic_block cfg_bb,
return true;
}
-avl_info::avl_info (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
-}
-
-avl_info::avl_info (rtx value_in, set_info *source_in)
- : m_value (value_in), m_source (source_in)
-{}
-
-bool
-avl_info::single_source_equal_p (const avl_info &other) const
-{
- set_info *set1 = m_source;
- set_info *set2 = other.get_source ();
- insn_info *insn1 = extract_single_source (set1);
- insn_info *insn2 = extract_single_source (set2);
- if (!insn1 || !insn2)
- return false;
- return source_equal_p (insn1, insn2);
-}
-
-bool
-avl_info::multiple_source_equal_p (const avl_info &other) const
-{
- /* When the def info is same in RTL_SSA namespace, it's safe
- to consider they are avl compatible. */
- if (m_source == other.get_source ())
- return true;
-
- /* We only consider handle PHI node. */
- if (!m_source->insn ()->is_phi () || !other.get_source ()->insn ()->is_phi ())
- return false;
-
- phi_info *phi1 = as_a<phi_info *> (m_source);
- phi_info *phi2 = as_a<phi_info *> (other.get_source ());
-
- if (phi1->is_degenerate () && phi2->is_degenerate ())
- {
- /* Degenerate PHI means the PHI node only have one input. */
-
- /* If both PHI nodes have the same single input in use list.
- We consider they are AVL compatible. */
- if (phi1->input_value (0) == phi2->input_value (0))
- return true;
- }
- /* TODO: We can support more optimization cases in the future. */
- return false;
-}
-
-avl_info &
-avl_info::operator= (const avl_info &other)
-{
- m_value = other.get_value ();
- m_source = other.get_source ();
- return *this;
-}
-
-bool
-avl_info::operator== (const avl_info &other) const
-{
- if (!m_value)
- return !other.get_value ();
- if (!other.get_value ())
- return false;
-
- if (GET_CODE (m_value) != GET_CODE (other.get_value ()))
- return false;
-
- /* Handle CONST_INT AVL. */
- if (CONST_INT_P (m_value))
- return INTVAL (m_value) == INTVAL (other.get_value ());
-
- /* Handle VLMAX AVL. */
- if (vlmax_avl_p (m_value))
- return vlmax_avl_p (other.get_value ());
- if (vlmax_avl_p (other.get_value ()))
- return false;
-
- /* If any source is undef value, we think they are not equal. */
- if (!m_source || !other.get_source ())
- return false;
-
- /* If both sources are single source (defined by a single real RTL)
- and their definitions are same. */
- if (single_source_equal_p (other))
- return true;
-
- return multiple_source_equal_p (other);
-}
-
-bool
-avl_info::operator!= (const avl_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-avl_info::has_non_zero_avl () const
-{
- if (has_avl_imm ())
- return INTVAL (get_value ()) > 0;
- if (has_avl_reg ())
- return vlmax_avl_p (get_value ());
- return false;
-}
-
-/* Initialize VL/VTYPE information. */
-vl_vtype_info::vl_vtype_info (avl_info avl_in, uint8_t sew_in,
- enum vlmul_type vlmul_in, uint8_t ratio_in,
- bool ta_in, bool ma_in)
- : m_avl (avl_in), m_sew (sew_in), m_vlmul (vlmul_in), m_ratio (ratio_in),
- m_ta (ta_in), m_ma (ma_in)
-{
- gcc_assert (valid_sew_p (m_sew) && "Unexpected SEW");
-}
-
-bool
-vl_vtype_info::operator== (const vl_vtype_info &other) const
-{
- return same_avl_p (other) && m_sew == other.get_sew ()
- && m_vlmul == other.get_vlmul () && m_ta == other.get_ta ()
- && m_ma == other.get_ma () && m_ratio == other.get_ratio ();
-}
-
-bool
-vl_vtype_info::operator!= (const vl_vtype_info &other) const
-{
- return !(*this == other);
-}
-
-bool
-vl_vtype_info::same_avl_p (const vl_vtype_info &other) const
-{
- /* We need to compare both RTL and SET. If both AVL are CONST_INT.
- For example, const_int 3 and const_int 4, we need to compare
- RTL. If both AVL are REG and their REGNO are same, we need to
- compare SET. */
- return get_avl () == other.get_avl ()
- && get_avl_source () == other.get_avl_source ();
-}
-
-bool
-vl_vtype_info::same_vtype_p (const vl_vtype_info &other) const
-{
- return get_sew () == other.get_sew () && get_vlmul () == other.get_vlmul ()
- && get_ta () == other.get_ta () && get_ma () == other.get_ma ();
-}
+/* This flags indicates the minimum demand of the vl and vtype values by the
+ RVV instruction. For example, DEMAND_RATIO_P indicates that this RVV
+ instruction only needs the SEW/LMUL ratio to remain the same, and does not
+ require SEW and LMUL to be fixed.
+ Therefore, if the former RVV instruction needs DEMAND_RATIO_P and the latter
+ instruction needs DEMAND_SEW_LMUL_P and its SEW/LMUL is the same as that of
+ the former instruction, then we can make the minimu demand of the former
+ instruction strict to DEMAND_SEW_LMUL_P, and its required SEW and LMUL are
+ the SEW and LMUL of the latter instruction, and the vsetvl instruction
+ generated according to the new demand can also be used for the latter
+ instruction, so there is no need to insert a separate vsetvl instruction for
+ the latter instruction. */
+enum demand_flags : unsigned
+{
+ DEMAND_EMPTY_P = 0,
+ DEMAND_SEW_P = 1 << 0,
+ DEMAND_LMUL_P = 1 << 1,
+ DEMAND_RATIO_P = 1 << 2,
+ DEMAND_GE_SEW_P = 1 << 3,
+ DEMAND_TAIL_POLICY_P = 1 << 4,
+ DEMAND_MASK_POLICY_P = 1 << 5,
+ DEMAND_AVL_P = 1 << 6,
+ DEMAND_NON_ZERO_AVL_P = 1 << 7,
+};
-bool
-vl_vtype_info::same_vlmax_p (const vl_vtype_info &other) const
-{
- return get_ratio () == other.get_ratio ();
-}
+/* We split the demand information into three parts. They are sew and lmul
+ related (sew_lmul_demand_type), tail and mask policy related
+ (policy_demand_type) and avl related (avl_demand_type). Then we define three
+ interfaces avaiable_with, compatible_with and merge_with. avaiable_with is
+ used to determine whether the two vsetvl infos prev_info and next_info are
+ available or not. If prev_info is available for next_info, it means that the
+ RVV insn corresponding to next_info on the path from prev_info to next_info
+ can be used without inserting a separate vsetvl instruction. compatible_with
+ is used to determine whether prev_info is compatible with next_info, and if
+ so, merge_with can be used to merge the stricter demand information from
+ next_info into prev_info so that prev_info becomes available to next_info.
+ */
-/* Compare the compatibility between Dem1 and Dem2.
- If Dem1 > Dem2, Dem1 has bigger compatibility then Dem2
- meaning Dem1 is easier be compatible with others than Dem2
- or Dem2 is stricter than Dem1.
- For example, Dem1 (demand SEW + LMUL) > Dem2 (demand RATIO). */
-bool
-vector_insn_info::operator>= (const vector_insn_info &other) const
+enum class sew_lmul_demand_type : unsigned
{
- if (support_relaxed_compatible_p (*this, other))
- {
- unsigned array_size = sizeof (unavailable_conds) / sizeof (demands_cond);
- /* Bypass AVL unavailable cases. */
- for (unsigned i = 2; i < array_size; i++)
- if (unavailable_conds[i].pair.match_cond_p (this->get_demands (),
- other.get_demands ())
- && unavailable_conds[i].incompatible_p (*this, other))
- return false;
- return true;
- }
-
- if (!other.compatible_p (static_cast<const vl_vtype_info &> (*this)))
- return false;
- if (!this->compatible_p (static_cast<const vl_vtype_info &> (other)))
- return true;
-
- if (*this == other)
- return true;
-
- for (const auto &cond : unavailable_conds)
- if (cond.pair.match_cond_p (this->get_demands (), other.get_demands ())
- && cond.incompatible_p (*this, other))
- return false;
-
- return true;
-}
+ sew_lmul = demand_flags::DEMAND_SEW_P | demand_flags::DEMAND_LMUL_P,
+ ratio_only = demand_flags::DEMAND_RATIO_P,
+ sew_only = demand_flags::DEMAND_SEW_P,
+ ge_sew = demand_flags::DEMAND_GE_SEW_P,
+ ratio_and_ge_sew
+ = demand_flags::DEMAND_RATIO_P | demand_flags::DEMAND_GE_SEW_P,
+};
-bool
-vector_insn_info::operator== (const vector_insn_info &other) const
+enum class policy_demand_type : unsigned
{
- gcc_assert (!uninit_p () && !other.uninit_p ()
- && "Uninitialization should not happen");
-
- /* Empty is only equal to another Empty. */
- if (empty_p ())
- return other.empty_p ();
- if (other.empty_p ())
- return empty_p ();
-
- /* Unknown is only equal to another Unknown. */
- if (unknown_p ())
- return other.unknown_p ();
- if (other.unknown_p ())
- return unknown_p ();
-
- for (size_t i = 0; i < NUM_DEMAND; i++)
- if (m_demands[i] != other.demand_p ((enum demand_type) i))
- return false;
-
- /* We should consider different INSN demands as different
- expression. Otherwise, we will be doing incorrect vsetvl
- elimination. */
- if (m_insn != other.get_insn ())
- return false;
-
- if (!same_avl_p (other))
- return false;
-
- /* If the full VTYPE is valid, check that it is the same. */
- return same_vtype_p (other);
-}
+ tail_mask_policy
+ = demand_flags::DEMAND_TAIL_POLICY_P | demand_flags::DEMAND_MASK_POLICY_P,
+ tail_policy_only = demand_flags::DEMAND_TAIL_POLICY_P,
+ mask_policy_only = demand_flags::DEMAND_MASK_POLICY_P,
+ ignore_policy = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (rtx_insn *rinsn)
+enum class avl_demand_type : unsigned
{
- *this = vector_insn_info ();
- if (!NONDEBUG_INSN_P (rinsn))
- return;
- if (optimize == 0 && !has_vtype_op (rinsn))
- return;
- gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
- m_state = VALID;
- extract_insn_cached (rinsn);
- rtx avl = ::get_avl (rinsn);
- m_avl = avl_info (avl, nullptr);
- m_sew = ::get_sew (rinsn);
- m_vlmul = ::get_vlmul (rinsn);
- m_ta = tail_agnostic_p (rinsn);
- m_ma = mask_agnostic_p (rinsn);
-}
+ avl = demand_flags::DEMAND_AVL_P,
+ non_zero_avl = demand_flags::DEMAND_NON_ZERO_AVL_P,
+ ignore_avl = demand_flags::DEMAND_EMPTY_P,
+};
-void
-vector_insn_info::parse_insn (insn_info *insn)
+class vsetvl_info
{
- *this = vector_insn_info ();
-
- /* Return if it is debug insn for the consistency with optimize == 0. */
- if (insn->is_debug_insn ())
- return;
+private:
+ insn_info *m_insn;
+ bb_info *m_bb;
+ rtx m_avl;
+ rtx m_vl;
+ set_info *m_avl_def;
+ uint8_t m_sew;
+ uint8_t m_max_sew;
+ vlmul_type m_vlmul;
+ uint8_t m_ratio;
+ bool m_ta;
+ bool m_ma;
+
+ sew_lmul_demand_type m_sew_lmul_demand;
+ policy_demand_type m_policy_demand;
+ avl_demand_type m_avl_demand;
+
+ enum class state_type
+ {
+ UNINITIALIZED,
+ VALID,
+ UNKNOWN,
+ EMPTY,
+ };
+ state_type m_state;
+
+ bool m_ignore;
+ bool change_vtype_only;
+ insn_info *m_read_vl_insn;
+ bool use_by_non_rvv_insn;
- /* We set it as unknown since we don't what will happen in CALL or ASM. */
- if (insn->is_call () || insn->is_asm ())
- {
- set_unknown ();
+public:
+ vsetvl_info ()
+ : m_insn (nullptr), m_bb (nullptr), m_avl (NULL_RTX), m_vl (NULL_RTX),
+ m_avl_def (nullptr), m_sew (0), m_max_sew (0), m_vlmul (LMUL_RESERVED),
+ m_ratio (0), m_ta (false), m_ma (false),
+ m_sew_lmul_demand (sew_lmul_demand_type::sew_lmul),
+ m_policy_demand (policy_demand_type::tail_mask_policy),
+ m_avl_demand (avl_demand_type::avl), m_state (state_type::UNINITIALIZED),
+ m_ignore (false), change_vtype_only (false), m_read_vl_insn (nullptr),
+ use_by_non_rvv_insn (false)
+ {}
+
+ vsetvl_info (insn_info *insn) : vsetvl_info () { parse_insn (insn); }
+
+ vsetvl_info (rtx_insn *insn) : vsetvl_info () { parse_insn (insn); }
+
+ void set_avl (rtx avl) { m_avl = avl; }
+ void set_vl (rtx vl) { m_vl = vl; }
+ void set_avl_def (set_info *avl_def) { m_avl_def = avl_def; }
+ void set_sew (uint8_t sew) { m_sew = sew; }
+ void set_vlmul (vlmul_type vlmul) { m_vlmul = vlmul; }
+ void set_ratio (uint8_t ratio) { m_ratio = ratio; }
+ void set_ta (bool ta) { m_ta = ta; }
+ void set_ma (bool ma) { m_ma = ma; }
+ void set_ignore () { m_ignore = true; }
+ void set_bb (bb_info *bb) { m_bb = bb; }
+ void set_max_sew (uint8_t max_sew) { m_max_sew = max_sew; }
+ void set_change_vtype_only () { change_vtype_only = true; }
+ void set_read_vl_insn (insn_info *insn) { m_read_vl_insn = insn; }
+
+ rtx get_avl () const { return m_avl; }
+ rtx get_vl () const { return m_vl; }
+ set_info *get_avl_def () const { return m_avl_def; }
+ uint8_t get_sew () const { return m_sew; }
+ vlmul_type get_vlmul () const { return m_vlmul; }
+ uint8_t get_ratio () const { return m_ratio; }
+ bool get_ta () const { return m_ta; }
+ bool get_ma () const { return m_ma; }
+ insn_info *get_insn () const { return m_insn; }
+ bool ignore_p () const { return m_ignore; }
+ bb_info *get_bb () const { return m_bb; }
+ uint8_t get_max_sew () const { return m_max_sew; }
+ insn_info *get_read_vl_insn () const { return m_read_vl_insn; }
+ bool use_by_non_rvv_insn_p () const { return use_by_non_rvv_insn; }
+
+ bool has_imm_avl () const { return m_avl && CONST_INT_P (m_avl); }
+ bool has_vlmax_avl () const { return vlmax_avl_p (m_avl); }
+ bool has_reg_avl () const
+ {
+ return m_avl && REG_P (m_avl) && !has_vlmax_avl ();
+ }
+ bool has_non_zero_avl () const
+ {
+ if (has_imm_avl ())
+ return INTVAL (m_avl) > 0;
+ return has_vlmax_avl ();
+ }
+ bool has_reg_vl () const
+ {
+ gcc_assert (!m_vl || REG_P (m_vl));
+ return m_vl && REG_P (m_vl);
+ }
+ bool has_same_ratio (const vsetvl_info &other) const
+ {
+ return get_ratio () == other.get_ratio ();
+ }
+ bool is_in_origin_bb () const { return get_insn ()->bb () == get_bb (); }
+ void update_avl (const vsetvl_info &other)
+ {
+ m_avl = other.get_avl ();
+ m_vl = other.get_vl ();
+ m_avl_def = other.get_avl_def ();
+ }
+
+ bool uninit_p () const { return m_state == state_type::UNINITIALIZED; }
+ bool valid_p () const { return m_state == state_type::VALID; }
+ bool unknown_p () const { return m_state == state_type::UNKNOWN; }
+ bool empty_p () const { return m_state == state_type::EMPTY; }
+ bool change_vtype_only_p () const { return change_vtype_only; }
+
+ void set_valid () { m_state = state_type::VALID; }
+ void set_unknown () { m_state = state_type::UNKNOWN; }
+ void set_empty () { m_state = state_type::EMPTY; }
+
+ void set_sew_lmul_demand (sew_lmul_demand_type demand)
+ {
+ m_sew_lmul_demand = demand;
+ }
+ void set_policy_demand (policy_demand_type demand)
+ {
+ m_policy_demand = demand;
+ }
+ void set_avl_demand (avl_demand_type demand) { m_avl_demand = demand; }
+
+ sew_lmul_demand_type get_sew_lmul_demand () const
+ {
+ return m_sew_lmul_demand;
+ }
+ policy_demand_type get_policy_demand () const { return m_policy_demand; }
+ avl_demand_type get_avl_demand () const { return m_avl_demand; }
+
+ void normalize_demand (unsigned demand_flags)
+ {
+ switch (demand_flags
+ & (DEMAND_SEW_P | DEMAND_LMUL_P | DEMAND_RATIO_P | DEMAND_GE_SEW_P))
+ {
+ case (unsigned) sew_lmul_demand_type::sew_lmul:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_lmul;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::sew_only:
+ m_sew_lmul_demand = sew_lmul_demand_type::sew_only;
+ break;
+ case (unsigned) sew_lmul_demand_type::ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ge_sew;
+ break;
+ case (unsigned) sew_lmul_demand_type::ratio_and_ge_sew:
+ m_sew_lmul_demand = sew_lmul_demand_type::ratio_and_ge_sew;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_TAIL_POLICY_P | DEMAND_MASK_POLICY_P))
+ {
+ case (unsigned) policy_demand_type::tail_mask_policy:
+ m_policy_demand = policy_demand_type::tail_mask_policy;
+ break;
+ case (unsigned) policy_demand_type::tail_policy_only:
+ m_policy_demand = policy_demand_type::tail_policy_only;
+ break;
+ case (unsigned) policy_demand_type::mask_policy_only:
+ m_policy_demand = policy_demand_type::mask_policy_only;
+ break;
+ case (unsigned) policy_demand_type::ignore_policy:
+ m_policy_demand = policy_demand_type::ignore_policy;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (demand_flags & (DEMAND_AVL_P | DEMAND_NON_ZERO_AVL_P))
+ {
+ case (unsigned) avl_demand_type::avl:
+ m_avl_demand = avl_demand_type::avl;
+ break;
+ case (unsigned) avl_demand_type::non_zero_avl:
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ break;
+ case (unsigned) avl_demand_type::ignore_avl:
+ m_avl_demand = avl_demand_type::ignore_avl;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ }
+
+ void parse_insn (rtx_insn *rinsn)
+ {
+ if (!NONDEBUG_INSN_P (rinsn))
return;
- }
-
- /* If this is something that updates VL/VTYPE that we don't know about, set
- the state to unknown. */
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
- && (find_access (insn->defs (), VL_REGNUM)
- || find_access (insn->defs (), VTYPE_REGNUM)))
- {
- set_unknown ();
+ if (optimize == 0 && !has_vtype_op (rinsn))
return;
- }
-
- if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
- return;
-
- /* Warning: This function has to work on both the lowered (i.e. post
- emit_local_forward_vsetvls) and pre-lowering forms. The main implication
- of this is that it can't use the value of a SEW, VL, or Policy operand as
- they might be stale after lowering. */
- vl_vtype_info::operator= (get_vl_vtype_info (insn));
- m_insn = insn;
- m_state = VALID;
- if (vector_config_insn_p (insn->rtl ()))
- {
- m_demands[DEMAND_AVL] = true;
- m_demands[DEMAND_RATIO] = true;
+ gcc_assert (!vsetvl_discard_result_insn_p (rinsn));
+ set_valid ();
+ extract_insn_cached (rinsn);
+ m_avl = ::get_avl (rinsn);
+ if (has_vlmax_avl () || vsetvl_insn_p (rinsn))
+ m_vl = ::get_vl (rinsn);
+ m_sew = ::get_sew (rinsn);
+ m_vlmul = ::get_vlmul (rinsn);
+ m_ta = tail_agnostic_p (rinsn);
+ m_ma = mask_agnostic_p (rinsn);
+ }
+
+ void parse_insn (insn_info *insn)
+ {
+ m_insn = insn;
+ m_bb = insn->bb ();
+ /* Return if it is debug insn for the consistency with optimize == 0. */
+ if (insn->is_debug_insn ())
return;
- }
-
- if (has_vl_op (insn->rtl ()))
- m_demands[DEMAND_AVL] = true;
-
- if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_RATIO] = true;
- else
- {
- /* TODO: By default, if it doesn't demand RATIO, we set it
- demand SEW && LMUL both. Some instructions may demand SEW
- only and ignore LMUL, will fix it later. */
- m_demands[DEMAND_SEW] = true;
- if (!ignore_vlmul_insn_p (insn->rtl ()))
- m_demands[DEMAND_LMUL] = true;
- }
-
- if (get_attr_ta (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_TAIL_POLICY] = true;
- if (get_attr_ma (insn->rtl ()) != INVALID_ATTRIBUTE)
- m_demands[DEMAND_MASK_POLICY] = true;
-
- if (vector_config_insn_p (insn->rtl ()))
- return;
-
- if (scalar_move_insn_p (insn->rtl ()))
- {
- if (m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
- if (m_ta)
- m_demands[DEMAND_GE_SEW] = true;
- }
-
- if (!m_avl.has_avl_reg () || vlmax_avl_p (get_avl ()) || !m_avl.get_source ())
- return;
- if (!m_avl.get_source ()->insn ()->is_real ()
- && !m_avl.get_source ()->insn ()->is_phi ())
- return;
-
- insn_info *def_insn = extract_single_source (m_avl.get_source ());
- if (!def_insn || !vsetvl_insn_p (def_insn->rtl ()))
- return;
-
- vector_insn_info new_info;
- new_info.parse_insn (def_insn);
- if (!same_vlmax_p (new_info) && !scalar_move_insn_p (insn->rtl ()))
- return;
-
- if (new_info.has_avl ())
- {
- if (new_info.has_avl_imm ())
- set_avl_info (avl_info (new_info.get_avl (), nullptr));
- else
- {
- if (vlmax_avl_p (new_info.get_avl ()))
- set_avl_info (avl_info (new_info.get_avl (), get_avl_source ()));
- else
- {
- /* Conservatively propagate non-VLMAX AVL of user vsetvl:
- 1. The user vsetvl should be same block with the rvv insn.
- 2. The user vsetvl is the only def insn of rvv insn.
- 3. The AVL is not modified between def-use chain.
- 4. The VL is only used by insn within EBB.
- */
- bool modified_p = false;
- for (insn_info *i = def_insn->next_nondebug_insn ();
- real_insn_and_same_bb_p (i, get_insn ()->bb ());
- i = i->next_nondebug_insn ())
- {
- /* Consider this following sequence:
-
- insn 1: vsetvli a5,a3,e8,mf4,ta,mu
- insn 2: vsetvli zero,a5,e32,m1,ta,ma
- ...
- vle32.v v1,0(a1)
- vsetvli a2,zero,e32,m1,ta,ma
- vadd.vv v1,v1,v1
- vsetvli zero,a5,e32,m1,ta,ma
- vse32.v v1,0(a0)
- ...
- insn 3: sub a3,a3,a5
- ...
-
- We can local AVL propagate "a3" from insn 1 to insn 2
- if no insns between insn 1 and insn 2 modify "a3 even
- though insn 3 modifies "a3".
- Otherwise, we can't perform local AVL propagation.
-
- Early break if we reach the insn 2. */
- if (!before_p (i, insn))
- break;
- if (find_access (i->defs (), REGNO (new_info.get_avl ())))
- {
- modified_p = true;
- break;
- }
- }
-
- bool has_live_out_use = false;
- for (use_info *use : m_avl.get_source ()->all_uses ())
- {
- if (use->is_live_out_use ())
- {
- has_live_out_use = true;
- break;
- }
- }
- if (!modified_p && !has_live_out_use
- && def_insn == m_avl.get_source ()->insn ()
- && m_insn->bb () == def_insn->bb ())
- set_avl_info (new_info.get_avl_info ());
- }
- }
- }
- if (scalar_move_insn_p (insn->rtl ()) && m_avl.has_non_zero_avl ())
- m_demands[DEMAND_NONZERO_AVL] = true;
-}
-
-bool
-vector_insn_info::compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
-
- for (const auto &cond : incompatible_conds)
- if (cond.dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::skip_avl_compatible_p (const vector_insn_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && other.valid_or_dirty_p ()
- && "Can't compare invalid demanded infos");
- unsigned array_size = sizeof (incompatible_conds) / sizeof (demands_cond);
- /* Bypass AVL incompatible cases. */
- for (unsigned i = 1; i < array_size; i++)
- if (incompatible_conds[i].dual_incompatible_p (*this, other))
- return false;
- return true;
-}
-
-bool
-vector_insn_info::compatible_avl_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other.get_avl_info ();
-}
-
-bool
-vector_insn_info::compatible_avl_p (const avl_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare AVL in unknown state");
- gcc_assert (demand_p (DEMAND_AVL) && "Can't compare AVL undemand state");
- if (!demand_p (DEMAND_AVL))
- return true;
- if (demand_p (DEMAND_NONZERO_AVL) && other.has_non_zero_avl ())
- return true;
- return get_avl_info () == other;
-}
-
-bool
-vector_insn_info::compatible_vtype_p (const vl_vtype_info &other) const
-{
- gcc_assert (valid_or_dirty_p () && "Can't compare invalid vl_vtype_info");
- gcc_assert (!unknown_p () && "Can't compare VTYPE in unknown state");
- if (demand_p (DEMAND_SEW))
- {
- if (!demand_p (DEMAND_GE_SEW) && m_sew != other.get_sew ())
- return false;
- if (demand_p (DEMAND_GE_SEW) && m_sew > other.get_sew ())
- return false;
- }
- if (demand_p (DEMAND_LMUL) && m_vlmul != other.get_vlmul ())
- return false;
- if (demand_p (DEMAND_RATIO) && m_ratio != other.get_ratio ())
- return false;
- if (demand_p (DEMAND_TAIL_POLICY) && m_ta != other.get_ta ())
- return false;
- if (demand_p (DEMAND_MASK_POLICY) && m_ma != other.get_ma ())
- return false;
- return true;
-}
-
-/* Determine whether the vector instructions requirements represented by
- Require are compatible with the previous vsetvli instruction represented
- by this. INSN is the instruction whose requirements we're considering. */
-bool
-vector_insn_info::compatible_p (const vl_vtype_info &curr_info) const
-{
- gcc_assert (!uninit_p () && "Can't handle uninitialized info");
- if (empty_p ())
- return false;
-
- /* Nothing is compatible with Unknown. */
- if (unknown_p ())
- return false;
-
- /* If the instruction doesn't need an AVLReg and the SEW matches, consider
- it compatible. */
- if (!demand_p (DEMAND_AVL))
- if (m_sew == curr_info.get_sew ())
- return true;
-
- return compatible_avl_p (curr_info) && compatible_vtype_p (curr_info);
-}
-
-bool
-vector_insn_info::available_p (const vector_insn_info &other) const
-{
- return *this >= other;
-}
-
-void
-vector_insn_info::fuse_avl (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- set_insn (info1.get_insn ());
- if (info1.demand_p (DEMAND_AVL))
- {
- if (info1.demand_p (DEMAND_NONZERO_AVL))
- {
- if (info2.demand_p (DEMAND_AVL)
- && !info2.demand_p (DEMAND_NONZERO_AVL))
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_AVL, true);
- set_demand (DEMAND_NONZERO_AVL, false);
- return;
- }
- }
- set_avl_info (info1.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info1.demand_p (DEMAND_NONZERO_AVL));
- }
- else
- {
- set_avl_info (info2.get_avl_info ());
- set_demand (DEMAND_NONZERO_AVL, info2.demand_p (DEMAND_NONZERO_AVL));
- }
- set_demand (DEMAND_AVL,
- info1.demand_p (DEMAND_AVL) || info2.demand_p (DEMAND_AVL));
-}
-
-void
-vector_insn_info::fuse_sew_lmul (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- /* We need to fuse sew && lmul according to demand info:
-
- 1. GE_SEW.
- 2. SEW.
- 3. LMUL.
- 4. RATIO. */
- if (same_sew_lmul_demand_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, info2.demand_p (DEMAND_SEW));
- set_demand (DEMAND_LMUL, info2.demand_p (DEMAND_LMUL));
- set_demand (DEMAND_RATIO, info2.demand_p (DEMAND_RATIO));
- set_demand (DEMAND_GE_SEW, info2.demand_p (DEMAND_GE_SEW));
- set_sew (info2.get_sew ());
- set_vlmul (info2.get_vlmul ());
- set_ratio (info2.get_ratio ());
+ /* We set it as unknown since we don't what will happen in CALL or ASM. */
+ if (insn->is_call () || insn->is_asm ())
+ {
+ set_unknown ();
+ return;
+ }
+
+ /* If this is something that updates VL/VTYPE that we don't know about, set
+ the state to unknown. */
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ())
+ && (find_access (insn->defs (), VL_REGNUM)
+ || find_access (insn->defs (), VTYPE_REGNUM)))
+ {
+ set_unknown ();
+ return;
+ }
+
+ if (!vector_config_insn_p (insn->rtl ()) && !has_vtype_op (insn->rtl ()))
+ /* uninitialized */
return;
- }
- for (const auto &rule : fuse_rules)
- {
- if (rule.pair.match_cond_p (info1.get_demands (), info2.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info1, info2));
- set_vlmul (rule.new_vlmul (info1, info2));
- set_ratio (rule.new_ratio (info1, info2));
- return;
- }
- if (rule.pair.match_cond_p (info2.get_demands (), info1.get_demands ()))
- {
- set_demand (DEMAND_SEW, rule.demand_sew_p);
- set_demand (DEMAND_LMUL, rule.demand_lmul_p);
- set_demand (DEMAND_RATIO, rule.demand_ratio_p);
- set_demand (DEMAND_GE_SEW, rule.demand_ge_sew_p);
- set_sew (rule.new_sew (info2, info1));
- set_vlmul (rule.new_vlmul (info2, info1));
- set_ratio (rule.new_ratio (info2, info1));
- return;
- }
- }
- gcc_unreachable ();
-}
-void
-vector_insn_info::fuse_tail_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info1.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else if (info2.demand_p (DEMAND_TAIL_POLICY))
- {
- set_ta (info2.get_ta ());
- demand (DEMAND_TAIL_POLICY);
- }
- else
- set_ta (get_default_ta ());
-}
-
-void
-vector_insn_info::fuse_mask_policy (const vector_insn_info &info1,
- const vector_insn_info &info2)
-{
- if (info1.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info1.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else if (info2.demand_p (DEMAND_MASK_POLICY))
- {
- set_ma (info2.get_ma ());
- demand (DEMAND_MASK_POLICY);
- }
- else
- set_ma (get_default_ma ());
-}
-
-vector_insn_info
-vector_insn_info::local_merge (const vector_insn_info &merge_info) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
- /* For local backward data flow, we always update INSN && AVL as the
- latest INSN and AVL so that we can keep track status of each INSN. */
- new_info.fuse_avl (merge_info, *this);
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-vector_insn_info
-vector_insn_info::global_merge (const vector_insn_info &merge_info,
- unsigned int bb_index) const
-{
- if (!vsetvl_insn_p (get_insn ()->rtl ()) && *this != merge_info)
- gcc_assert (this->compatible_p (merge_info)
- && "Can't merge incompatible demanded infos");
-
- vector_insn_info new_info;
- new_info.set_valid ();
-
- /* For global data flow, we should keep original INSN and AVL if they
- valid since we should keep the life information of each block.
-
- For example:
- bb 0 -> bb 1.
- We should keep INSN && AVL of bb 1 since we will eventually emit
- vsetvl instruction according to INSN and AVL of bb 1. */
- new_info.fuse_avl (*this, merge_info);
- /* Recompute the AVL source whose block index is equal to BB_INDEX. */
- if (new_info.get_avl_source ()
- && new_info.get_avl_source ()->insn ()->is_phi ()
- && new_info.get_avl_source ()->bb ()->index () != bb_index)
- {
- hash_set<set_info *> sets
- = get_all_sets (new_info.get_avl_source (), true, true, true);
- new_info.set_avl_source (nullptr);
- bool can_find_set_p = false;
- set_info *first_set = nullptr;
- for (set_info *set : sets)
- {
- if (!first_set)
- first_set = set;
- if (set->bb ()->index () == bb_index)
- {
- gcc_assert (!can_find_set_p);
- new_info.set_avl_source (set);
- can_find_set_p = true;
- }
- }
- if (!can_find_set_p && sets.elements () == 1
- && first_set->insn ()->is_real ())
- new_info.set_avl_source (first_set);
- }
-
- /* Make sure VLMAX AVL always has a set_info the get VL. */
- if (vlmax_avl_p (new_info.get_avl ()))
- {
- if (this->get_avl_source ())
- new_info.set_avl_source (this->get_avl_source ());
- else
- {
- gcc_assert (merge_info.get_avl_source ());
- new_info.set_avl_source (merge_info.get_avl_source ());
- }
- }
-
- new_info.fuse_sew_lmul (*this, merge_info);
- new_info.fuse_tail_policy (*this, merge_info);
- new_info.fuse_mask_policy (*this, merge_info);
- return new_info;
-}
-
-/* Wrapper helps to return the AVL or VL operand for the
- vector_insn_info. Return AVL if the AVL is not VLMAX.
- Otherwise, return the VL operand. */
-rtx
-vector_insn_info::get_avl_or_vl_reg (void) const
-{
- gcc_assert (has_avl_reg ());
- if (!vlmax_avl_p (get_avl ()))
- return get_avl ();
-
- rtx_insn *rinsn = get_insn ()->rtl ();
- if (has_vl_op (rinsn) || vsetvl_insn_p (rinsn))
- {
- rtx vl = ::get_vl (rinsn);
- /* For VLMAX, we should make sure we get the
- REG to emit 'vsetvl VL,zero' since the 'VL'
- should be the REG according to RVV ISA. */
- if (REG_P (vl))
- return vl;
- }
-
- /* We always has avl_source if it is VLMAX AVL. */
- gcc_assert (get_avl_source ());
- return get_avl_reg_rtx ();
-}
+ set_valid ();
+
+ m_avl = ::get_avl (insn->rtl ());
+ if (m_avl)
+ {
+ if (vsetvl_insn_p (insn->rtl ()) || has_vlmax_avl ())
+ m_vl = ::get_vl (insn->rtl ());
+
+ if (has_reg_avl ())
+ m_avl_def = find_access (insn->uses (), REGNO (m_avl))->def ();
+ }
+
+ m_sew = ::get_sew (insn->rtl ());
+ m_vlmul = ::get_vlmul (insn->rtl ());
+ m_ratio = get_attr_ratio (insn->rtl ());
+ /* when get_attr_ratio is invalid, this kind of instructions
+ doesn't care about ratio. However, we still need this value
+ in demand info backward analysis. */
+ if (m_ratio == INVALID_ATTRIBUTE)
+ m_ratio = calculate_ratio (m_sew, m_vlmul);
+ m_ta = tail_agnostic_p (insn->rtl ());
+ m_ma = mask_agnostic_p (insn->rtl ());
+
+ /* If merge operand is undef value, we prefer agnostic. */
+ int merge_op_idx = get_attr_merge_op_idx (insn->rtl ());
+ if (merge_op_idx != INVALID_ATTRIBUTE
+ && satisfies_constraint_vu (recog_data.operand[merge_op_idx]))
+ {
+ m_ta = true;
+ m_ma = true;
+ }
+
+ /* Determine the demand info of the RVV insn. */
+ m_max_sew = get_max_int_sew ();
+ unsigned demand_flags = 0;
+ if (vector_config_insn_p (insn->rtl ()))
+ {
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ }
+ else
+ {
+ if (has_vl_op (insn->rtl ()))
+ {
+ if (scalar_move_insn_p (insn->rtl ()))
+ {
+ /* If the avl for vmv.s.x comes from the vsetvl instruction, we
+ don't know if the avl is non-zero, so it is set to
+ DEMAND_AVL_P for now. it may be corrected to
+ DEMAND_NON_ZERO_AVL_P later when more information is
+ available.
+ */
+ if (has_non_zero_avl ())
+ demand_flags |= demand_flags::DEMAND_NON_ZERO_AVL_P;
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_AVL_P;
+ }
-bool
-vector_insn_info::update_fault_first_load_avl (insn_info *insn)
-{
- // Update AVL to vl-output of the fault first load.
- const insn_info *read_vl = get_forward_read_vl_insn (insn);
- if (read_vl)
- {
- rtx vl = SET_DEST (PATTERN (read_vl->rtl ()));
- def_info *def = find_access (read_vl->defs (), REGNO (vl));
- set_info *set = safe_dyn_cast<set_info *> (def);
- set_avl_info (avl_info (vl, set));
- set_insn (insn);
- return true;
- }
- return false;
-}
+ if (get_attr_ratio (insn->rtl ()) != INVALID_ATTRIBUTE)
+ demand_flags |= demand_flags::DEMAND_RATIO_P;
+ else
+ {
+ if (scalar_move_insn_p (insn->rtl ()) && m_ta)
+ {
+ demand_flags |= demand_flags::DEMAND_GE_SEW_P;
+ m_max_sew = get_attr_type (insn->rtl ()) == TYPE_VFMOVFV
+ ? get_max_float_sew ()
+ : get_max_int_sew ();
+ }
+ else
+ demand_flags |= demand_flags::DEMAND_SEW_P;
+
+ if (!ignore_vlmul_insn_p (insn->rtl ()))
+ demand_flags |= demand_flags::DEMAND_LMUL_P;
+ }
-static const char *
-vlmul_to_str (vlmul_type vlmul)
-{
- switch (vlmul)
- {
- case LMUL_1:
- return "m1";
- case LMUL_2:
- return "m2";
- case LMUL_4:
- return "m4";
- case LMUL_8:
- return "m8";
- case LMUL_RESERVED:
- return "INVALID LMUL";
- case LMUL_F8:
- return "mf8";
- case LMUL_F4:
- return "mf4";
- case LMUL_F2:
- return "mf2";
+ if (!m_ta)
+ demand_flags |= demand_flags::DEMAND_TAIL_POLICY_P;
+ if (!m_ma)
+ demand_flags |= demand_flags::DEMAND_MASK_POLICY_P;
+ }
+
+ normalize_demand (demand_flags);
+
+ /* Optimize AVL from the vsetvl instruction. */
+ insn_info *def_insn = extract_single_source (get_avl_def ());
+ if (def_insn && vsetvl_insn_p (def_insn->rtl ()))
+ {
+ vsetvl_info def_info = vsetvl_info (def_insn);
+ if ((scalar_move_insn_p (insn->rtl ())
+ || def_info.get_ratio () == get_ratio ())
+ && (def_info.has_vlmax_avl () || def_info.has_imm_avl ()))
+ {
+ update_avl (def_info);
+ if (scalar_move_insn_p (insn->rtl ()) && has_non_zero_avl ())
+ m_avl_demand = avl_demand_type::non_zero_avl;
+ }
+ }
+
+ /* Determine if dest operand(vl) has been used by non-RVV instructions. */
+ if (has_reg_vl ())
+ {
+ const hash_set<use_info *> vl_uses
+ = get_all_real_uses (get_insn (), REGNO (get_vl ()));
+ for (use_info *use : vl_uses)
+ {
+ gcc_assert (use->insn ()->is_real ());
+ rtx_insn *rinsn = use->insn ()->rtl ();
+ if (!has_vl_op (rinsn)
+ || count_regno_occurrences (rinsn, REGNO (get_vl ())) != 1)
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ rtx avl = ::get_avl (rinsn);
+ if (!avl || REGNO (get_vl ()) != REGNO (avl))
+ {
+ use_by_non_rvv_insn = true;
+ break;
+ }
+ }
+ }
- default:
+ /* Collect the read vl insn for the fault-only-first rvv loads. */
+ if (fault_first_load_p (insn->rtl ()))
+ {
+ for (insn_info *i = insn->next_nondebug_insn ();
+ i->bb () == insn->bb (); i = i->next_nondebug_insn ())
+ {
+ if (find_access (i->defs (), VL_REGNUM))
+ break;
+ if (i->rtl () && read_vl_insn_p (i->rtl ()))
+ {
+ m_read_vl_insn = i;
+ break;
+ }
+ }
+ }
+ }
+
+ bool operator== (const vsetvl_info &other) const
+ {
+ gcc_assert (!uninit_p () && !other.uninit_p ()
+ && "Uninitialization should not happen");
+
+ if (empty_p ())
+ return other.empty_p ();
+ if (unknown_p ())
+ return other.unknown_p ();
+
+ return get_insn () == other.get_insn () && get_bb () == other.get_bb ()
+ && get_avl () == other.get_avl () && get_vl () == other.get_vl ()
+ && get_avl_def () == other.get_avl_def ()
+ && get_sew () == other.get_sew ()
+ && get_vlmul () == other.get_vlmul () && get_ta () == other.get_ta ()
+ && get_ma () == other.get_ma ()
+ && get_avl_demand () == other.get_avl_demand ()
+ && get_sew_lmul_demand () == other.get_sew_lmul_demand ()
+ && get_policy_demand () == other.get_policy_demand ();
+ }
+
+ void dump (FILE *file, const char *indent = "") const
+ {
+ if (uninit_p ())
+ {
+ fprintf (file, "UNINITIALIZED.\n");
+ return;
+ }
+ else if (unknown_p ())
+ {
+ fprintf (file, "UNKNOWN.\n");
+ return;
+ }
+ else if (empty_p ())
+ {
+ fprintf (file, "EMPTY.\n");
+ return;
+ }
+ else if (valid_p ())
+ fprintf (file, "VALID (insn %u, bb %u)%s\n", get_insn ()->uid (),
+ get_bb ()->index (), ignore_p () ? " (ignore)" : "");
+ else
gcc_unreachable ();
- }
-}
-
-static const char *
-policy_to_str (bool agnostic_p)
-{
- return agnostic_p ? "agnostic" : "undisturbed";
-}
-void
-vector_insn_info::dump (FILE *file) const
-{
- fprintf (file, "[");
- if (uninit_p ())
- fprintf (file, "UNINITIALIZED,");
- else if (valid_p ())
- fprintf (file, "VALID,");
- else if (unknown_p ())
- fprintf (file, "UNKNOWN,");
- else if (empty_p ())
- fprintf (file, "EMPTY,");
- else
- fprintf (file, "DIRTY,");
-
- fprintf (file, "Demand field={%d(VL),", demand_p (DEMAND_AVL));
- fprintf (file, "%d(DEMAND_NONZERO_AVL),", demand_p (DEMAND_NONZERO_AVL));
- fprintf (file, "%d(SEW),", demand_p (DEMAND_SEW));
- fprintf (file, "%d(DEMAND_GE_SEW),", demand_p (DEMAND_GE_SEW));
- fprintf (file, "%d(LMUL),", demand_p (DEMAND_LMUL));
- fprintf (file, "%d(RATIO),", demand_p (DEMAND_RATIO));
- fprintf (file, "%d(TAIL_POLICY),", demand_p (DEMAND_TAIL_POLICY));
- fprintf (file, "%d(MASK_POLICY)}\n", demand_p (DEMAND_MASK_POLICY));
-
- fprintf (file, "AVL=");
- print_rtl_single (file, get_avl ());
- fprintf (file, "SEW=%d,", get_sew ());
- fprintf (file, "VLMUL=%s,", vlmul_to_str (get_vlmul ()));
- fprintf (file, "RATIO=%d,", get_ratio ());
- fprintf (file, "TAIL_POLICY=%s,", policy_to_str (get_ta ()));
- fprintf (file, "MASK_POLICY=%s", policy_to_str (get_ma ()));
- fprintf (file, "]\n");
-
- if (valid_p ())
- {
- if (get_insn ())
- {
- fprintf (file, "The real INSN=");
- print_rtl_single (file, get_insn ()->rtl ());
- }
- }
-}
+ fprintf (file, "%sDemand fields:", indent);
+ if (m_sew_lmul_demand == sew_lmul_demand_type::sew_lmul)
+ fprintf (file, " demand_sew_lmul");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_only)
+ fprintf (file, " demand_ratio_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::sew_only)
+ fprintf (file, " demand_sew_only");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ge_sew)
+ fprintf (file, " demand_ge_sew");
+ else if (m_sew_lmul_demand == sew_lmul_demand_type::ratio_and_ge_sew)
+ fprintf (file, " demand_ratio_and_ge_sew");
+
+ if (m_policy_demand == policy_demand_type::tail_mask_policy)
+ fprintf (file, " demand_tail_mask_policy");
+ else if (m_policy_demand == policy_demand_type::tail_policy_only)
+ fprintf (file, " demand_tail_policy_only");
+ else if (m_policy_demand == policy_demand_type::mask_policy_only)
+ fprintf (file, " demand_mask_policy_only");
+
+ if (m_avl_demand == avl_demand_type::avl)
+ fprintf (file, " demand_avl");
+ else if (m_avl_demand == avl_demand_type::non_zero_avl)
+ fprintf (file, " demand_non_zero_avl");
+ fprintf (file, "\n");
+
+ fprintf (file, "%sSEW=%d, ", indent, get_sew ());
+ fprintf (file, "VLMUL=%s, ", vlmul_to_str (get_vlmul ()));
+ fprintf (file, "RATIO=%d, ", get_ratio ());
+ fprintf (file, "MAX_SEW=%d\n", get_max_sew ());
+
+ fprintf (file, "%sTAIL_POLICY=%s, ", indent, policy_to_str (get_ta ()));
+ fprintf (file, "MASK_POLICY=%s\n", policy_to_str (get_ma ()));
+
+ fprintf (file, "%sAVL=", indent);
+ print_rtl_single (file, get_avl ());
+ fprintf (file, "%sVL=", indent);
+ print_rtl_single (file, get_vl ());
+ if (change_vtype_only_p ())
+ fprintf (file, "%schange vtype only\n", indent);
+ if (get_read_vl_insn ())
+ fprintf (file, "%sread_vl_insn: insn %u\n", indent,
+ get_read_vl_insn ()->uid ());
+ if (use_by_non_rvv_insn_p ())
+ fprintf (file, "%suse_by_non_rvv_insn=true\n", indent);
+ }
+};
vector_infos_manager::vector_infos_manager ()
{
@@ -82,267 +82,6 @@ enum def_type
CLOBBER_DEF = 1 << 4
};
-/* AVL info for RVV instruction. Most RVV instructions have AVL operand in
- implicit dependency. The AVL comparison between 2 RVV instructions is
- very important since it affects our decision whether we should insert
- a vsetvl instruction in this situation. AVL operand of all RVV instructions
- can only be either a const_int value with < 32 or a reg value which can be
- define by either a real RTL instruction or a PHI instruction. So we need a
- standalone method to define AVL comparison and we can not simpily use
- operator "==" to compare 2 RTX value since it's to strict which will make
- use miss a lot of optimization opportunities. This method handle these
- following cases:
-
- - Background:
- Insert-vsetvl PASS is working after RA.
-
- - Terminology:
- - pr: Pseudo-register.
- - hr: Hardware-register.
-
- - Case 1:
-
- Before RA:
- li pr138,13
- insn1 (implicit depend on pr138).
- li pr138,14
- insn2 (implicit depend on pr139).
-
- After RA:
- li hr5,13
- insn1 (implicit depend on hr5).
- li hr5,14
- insn2 (implicit depend on hr5).
-
- Correct IR after vsetvl PASS:
- li hr5,13
- vsetvl1 zero,hr5....
- insn1 (implicit depend on hr5).
- li hr5,14
- vsetvl2 zero,hr5....
- insn2 (implicit depend on hr5).
-
- In this case, both insn1 and insn2 are using hr5 as the same AVL.
- If we use "rtx_equal_p" or "REGNO (AVL1) == REGNO (AVL)", we will end
- up with missing the vsetvl2 instruction which creates wrong result.
-
- Note: Using "==" operator to compare 2 AVL RTX strictly can fix this
- issue. However, it is a too strict comparison method since not all member
- variables in RTX data structure are not neccessary to be the same. It will
- make us miss a lot of optimization opportunities.
-
- - Case 2:
-
- After RA:
- bb 0:
- li hr5,13
- bb 1:
- li hr5,14
- bb2:
- insn1 (implicit depend on hr5).
- insn2 (implicit depend on hr5).
-
- In this case, we may end up with different AVL RTX and produce redundant
- vsetvl instruction.
-
- VALUE is the implicit dependency in each RVV instruction.
- SOURCE is the source definition information of AVL operand. */
-class avl_info
-{
-private:
- rtx m_value;
- rtl_ssa::set_info *m_source;
-
-public:
- avl_info () : m_value (NULL_RTX), m_source (nullptr) {}
- avl_info (const avl_info &);
- avl_info (rtx, rtl_ssa::set_info *);
- rtx get_value () const { return m_value; }
- rtl_ssa::set_info *get_source () const { return m_source; }
- void set_source (rtl_ssa::set_info *set) { m_source = set; }
- bool single_source_equal_p (const avl_info &) const;
- bool multiple_source_equal_p (const avl_info &) const;
- avl_info &operator= (const avl_info &);
- bool operator== (const avl_info &) const;
- bool operator!= (const avl_info &) const;
-
- bool has_avl_imm () const
- {
- return get_value () && CONST_INT_P (get_value ());
- }
- bool has_avl_reg () const { return get_value () && REG_P (get_value ()); }
- bool has_avl_no_reg () const { return !get_value (); }
- bool has_non_zero_avl () const;
- bool has_avl () const { return get_value (); }
-};
-
-/* Basic structure to save VL/VTYPE information. */
-struct vl_vtype_info
-{
-protected:
- /* AVL can be either register or const_int. */
- avl_info m_avl;
- /* Fields from VTYPE. The VTYPE checking depend on the flag
- dem_* before. */
- uint8_t m_sew;
- riscv_vector::vlmul_type m_vlmul;
- uint8_t m_ratio;
- bool m_ta;
- bool m_ma;
-
-public:
- void set_sew (uint8_t sew) { m_sew = sew; }
- void set_vlmul (riscv_vector::vlmul_type vlmul) { m_vlmul = vlmul; }
- void set_ratio (uint8_t ratio) { m_ratio = ratio; }
- void set_ta (bool ta) { m_ta = ta; }
- void set_ma (bool ma) { m_ma = ma; }
-
- vl_vtype_info ()
- : m_avl (avl_info ()), m_sew (0), m_vlmul (riscv_vector::LMUL_RESERVED),
- m_ratio (0), m_ta (0), m_ma (0)
- {}
- vl_vtype_info (const vl_vtype_info &) = default;
- vl_vtype_info &operator= (const vl_vtype_info &) = default;
- vl_vtype_info (avl_info, uint8_t, riscv_vector::vlmul_type, uint8_t, bool,
- bool);
-
- bool operator== (const vl_vtype_info &) const;
- bool operator!= (const vl_vtype_info &) const;
-
- bool has_avl_imm () const { return m_avl.has_avl_imm (); }
- bool has_avl_reg () const { return m_avl.has_avl_reg (); }
- bool has_avl_no_reg () const { return m_avl.has_avl_no_reg (); }
- bool has_non_zero_avl () const { return m_avl.has_non_zero_avl (); };
- bool has_avl () const { return m_avl.has_avl (); }
-
- rtx get_avl () const { return m_avl.get_value (); }
- const avl_info &get_avl_info () const { return m_avl; }
- rtl_ssa::set_info *get_avl_source () const { return m_avl.get_source (); }
- void set_avl_source (rtl_ssa::set_info *set) { m_avl.set_source (set); }
- void set_avl_info (const avl_info &avl) { m_avl = avl; }
- uint8_t get_sew () const { return m_sew; }
- riscv_vector::vlmul_type get_vlmul () const { return m_vlmul; }
- uint8_t get_ratio () const { return m_ratio; }
- bool get_ta () const { return m_ta; }
- bool get_ma () const { return m_ma; }
-
- bool same_avl_p (const vl_vtype_info &) const;
- bool same_vtype_p (const vl_vtype_info &) const;
- bool same_vlmax_p (const vl_vtype_info &) const;
-};
-
-class vector_insn_info : public vl_vtype_info
-{
-private:
- enum state_type
- {
- UNINITIALIZED,
- VALID,
- UNKNOWN,
- EMPTY,
-
- /* The block is polluted as containing VSETVL instruction during dem
- backward propagation to gain better LCM optimization even though
- such VSETVL instruction is not really emit yet during this time. */
- DIRTY,
- };
-
- enum state_type m_state;
-
- bool m_demands[NUM_DEMAND];
-
- /* TODO: Assume INSN1 = INSN holding of definition of AVL.
- INSN2 = INSN that is inserted a vsetvl insn before.
- We may need to add a new member to save INSN of holding AVL.
- m_insn is holding the INSN that is inserted a vsetvl insn before in
- Phase 2. Ideally, most of the time INSN1 == INSN2. However, considering
- such case:
-
- vmv.x.s (INSN2)
- vle8.v (INSN1)
-
- If these 2 instructions are compatible, we should only issue a vsetvl INSN
- (with AVL included) before vmv.x.s, but vmv.x.s is not the INSN holding the
- definition of AVL. */
- rtl_ssa::insn_info *m_insn;
-
- friend class vector_infos_manager;
-
-public:
- vector_insn_info ()
- : vl_vtype_info (), m_state (UNINITIALIZED), m_demands{false},
- m_insn (nullptr)
- {}
-
- /* Parse the instruction to get VL/VTYPE information and demanding
- * information. */
- /* This is only called by simple_vsetvl subroutine when optimize == 0.
- Since RTL_SSA can not be enabled when optimize == 0, we don't initialize
- the m_insn. */
- void parse_insn (rtx_insn *);
- /* This is only called by lazy_vsetvl subroutine when optimize > 0.
- We use RTL_SSA framework to initialize the insn_info. */
- void parse_insn (rtl_ssa::insn_info *);
-
- bool operator>= (const vector_insn_info &) const;
- bool operator== (const vector_insn_info &) const;
-
- bool uninit_p () const { return m_state == UNINITIALIZED; }
- bool valid_p () const { return m_state == VALID; }
- bool unknown_p () const { return m_state == UNKNOWN; }
- bool empty_p () const { return m_state == EMPTY; }
- bool dirty_p () const { return m_state == DIRTY; }
- bool valid_or_dirty_p () const
- {
- return m_state == VALID || m_state == DIRTY;
- }
- bool available_p (const vector_insn_info &) const;
-
- static vector_insn_info get_unknown ()
- {
- vector_insn_info info;
- info.set_unknown ();
- return info;
- }
-
- void set_valid () { m_state = VALID; }
- void set_unknown () { m_state = UNKNOWN; }
- void set_empty () { m_state = EMPTY; }
- void set_dirty () { m_state = DIRTY; }
- void set_insn (rtl_ssa::insn_info *insn) { m_insn = insn; }
-
- bool demand_p (enum demand_type type) const { return m_demands[type]; }
- void demand (enum demand_type type) { m_demands[type] = true; }
- void set_demand (enum demand_type type, bool value)
- {
- m_demands[type] = value;
- }
- void fuse_avl (const vector_insn_info &, const vector_insn_info &);
- void fuse_sew_lmul (const vector_insn_info &, const vector_insn_info &);
- void fuse_tail_policy (const vector_insn_info &, const vector_insn_info &);
- void fuse_mask_policy (const vector_insn_info &, const vector_insn_info &);
-
- bool compatible_p (const vector_insn_info &) const;
- bool skip_avl_compatible_p (const vector_insn_info &) const;
- bool compatible_avl_p (const vl_vtype_info &) const;
- bool compatible_avl_p (const avl_info &) const;
- bool compatible_vtype_p (const vl_vtype_info &) const;
- bool compatible_p (const vl_vtype_info &) const;
- vector_insn_info local_merge (const vector_insn_info &) const;
- vector_insn_info global_merge (const vector_insn_info &, unsigned int) const;
-
- rtl_ssa::insn_info *get_insn () const { return m_insn; }
- const bool *get_demands (void) const { return m_demands; }
- rtx get_avl_or_vl_reg (void) const;
- rtx get_avl_reg_rtx (void) const
- {
- return gen_rtx_REG (Pmode, get_avl_source ()->regno ());
- }
- bool update_fault_first_load_avl (rtl_ssa::insn_info *);
-
- void dump (FILE *) const;
-};
-
struct vector_block_info
{
/* The local_dem vector insn_info of the block. */