diff mbox series

[V9,10/10] arm64/perf: Implement branch records save on PMU IRQ

Message ID 20230315051444.1683170-11-anshuman.khandual@arm.com
State New
Headers
Series arm64/perf: Enable branch stack sampling |

Commit Message

Anshuman Khandual March 15, 2023, 5:14 a.m. UTC 
  This modifies armv8pmu_branch_read() to concatenate live entries along with
task context stored entries and then process the resultant buffer to create
perf branch entry array for perf_sample_data. It follows the same principle
like task sched out.

Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: linux-arm-kernel@lists.infradead.org
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
---
 arch/arm64/kernel/brbe.c | 75 +++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 47 deletions(-)

Comments

James Clark May 23, 2023, 2:39 p.m. UTC | #1 
On 15/03/2023 05:14, Anshuman Khandual wrote:
> This modifies armv8pmu_branch_read() to concatenate live entries along with
> task context stored entries and then process the resultant buffer to create
> perf branch entry array for perf_sample_data. It follows the same principle
> like task sched out.
> 
> Cc: Catalin Marinas <catalin.marinas@arm.com>
> Cc: Will Deacon <will@kernel.org>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: linux-arm-kernel@lists.infradead.org
> Cc: linux-kernel@vger.kernel.org
> Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
> ---

[...]

>  void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
>  {
>  	struct brbe_hw_attr *brbe_attr = (struct brbe_hw_attr *)cpuc->percpu_pmu->private;
> +	struct arm64_perf_task_context *task_ctx = event->pmu_ctx->task_ctx_data;
> +	struct brbe_regset live[BRBE_MAX_ENTRIES];
> +	int nr_live, nr_store;
>  	u64 brbfcr, brbcr;
> -	int idx, loop1_idx1, loop1_idx2, loop2_idx1, loop2_idx2, count;
>  
>  	brbcr = read_sysreg_s(SYS_BRBCR_EL1);
>  	brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
> @@ -739,36 +743,13 @@ void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
>  	write_sysreg_s(brbfcr | BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
>  	isb();
>  
> -	/* Determine the indices for each loop */
> -	loop1_idx1 = BRBE_BANK0_IDX_MIN;
> -	if (brbe_attr->brbe_nr <= BRBE_BANK_MAX_ENTRIES) {
> -		loop1_idx2 = brbe_attr->brbe_nr - 1;
> -		loop2_idx1 = BRBE_BANK1_IDX_MIN;
> -		loop2_idx2 = BRBE_BANK0_IDX_MAX;
> -	} else {
> -		loop1_idx2 = BRBE_BANK0_IDX_MAX;
> -		loop2_idx1 = BRBE_BANK1_IDX_MIN;
> -		loop2_idx2 = brbe_attr->brbe_nr - 1;
> -	}
> -
> -	/* Loop through bank 0 */
> -	select_brbe_bank(BRBE_BANK_IDX_0);
> -	for (idx = 0, count = loop1_idx1; count <= loop1_idx2; idx++, count++) {
> -		if (!capture_branch_entry(cpuc, event, idx))
> -			goto skip_bank_1;
> -	}
> -
> -	/* Loop through bank 1 */
> -	select_brbe_bank(BRBE_BANK_IDX_1);
> -	for (count = loop2_idx1; count <= loop2_idx2; idx++, count++) {
> -		if (!capture_branch_entry(cpuc, event, idx))
> -			break;
> -	}
> -
> -skip_bank_1:
> -	cpuc->branches->branch_stack.nr = idx;
> -	cpuc->branches->branch_stack.hw_idx = -1ULL;
> +	nr_live = capture_brbe_regset(brbe_attr, live);
> +	nr_store = task_ctx->nr_brbe_records;
> +	nr_store = stitch_stored_live_entries(task_ctx->store, live, nr_store,
> +					      nr_live, brbe_attr->brbe_nr);
> +	process_branch_entries(cpuc, event, task_ctx->store, nr_store);

Hi Anshuman,

With the following command I get a crash:

  perf record --branch-filter any,save_type -a -- ls

[  101.171822] Unable to handle kernel NULL pointer dereference at
virtual address 0000000000000600
...
[145380.414654] Call trace:
[145380.414739]  armv8pmu_branch_read+0x7c/0x578
[145380.414895]  armv8pmu_handle_irq+0x104/0x1c0
[145380.415043]  armpmu_dispatch_irq+0x38/0x70
[145380.415209]  __handle_irq_event_percpu+0x124/0x3b8
[145380.415392]  handle_irq_event+0x54/0xc8
[145380.415567]  handle_fasteoi_irq+0x100/0x1e0
[145380.415718]  generic_handle_domain_irq+0x38/0x58
[145380.415895]  gic_handle_irq+0x5c/0x130
[145380.416025]  call_on_irq_stack+0x24/0x58
[145380.416173]  el1_interrupt+0x74/0xc0
[145380.416321]  el1h_64_irq_handler+0x18/0x28
[145380.416475]  el1h_64_irq+0x64/0x68
[145380.416604]  smp_call_function_single+0xe8/0x1f0
[145380.416745]  event_function_call+0xbc/0x1c8
[145380.416919]  _perf_event_enable+0x84/0xa0
[145380.417069]  perf_ioctl+0xe8/0xd68
[145380.417204]  __arm64_sys_ioctl+0x9c/0xe0
[145380.417353]  invoke_syscall+0x4c/0x120
[145380.417523]  el0_svc_common+0xd0/0x120
[145380.417693]  do_el0_svc+0x3c/0xb8
[145380.417859]  el0_svc+0x50/0xc0
[145380.418004]  el0t_64_sync_handler+0x84/0xf0
[145380.418160]  el0t_64_sync+0x190/0x198

When using --branch-filter any,u without -a it seems to be fine so could
be that task_ctx is null in per-cpu mode, or something to do with the
userspace only flag?

I'm also wondering if it's possible to collapse some of the last 5
commits? They seem to mostly modify things in brbe.c which is a new file
so the history probably isn't important at this point it just makes it a
bit harder to review.

>  	process_branch_aborts(cpuc);
> +	task_ctx->nr_brbe_records = 0;
>  
>  	/* Unpause the buffer */
>  	write_sysreg_s(brbfcr & ~BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
James Clark May 23, 2023, 2:51 p.m. UTC | #2 
On 23/05/2023 15:39, James Clark wrote:
> 
> 
> On 15/03/2023 05:14, Anshuman Khandual wrote:
>> This modifies armv8pmu_branch_read() to concatenate live entries along with
>> task context stored entries and then process the resultant buffer to create
>> perf branch entry array for perf_sample_data. It follows the same principle
>> like task sched out.
>>
>> Cc: Catalin Marinas <catalin.marinas@arm.com>
>> Cc: Will Deacon <will@kernel.org>
>> Cc: Mark Rutland <mark.rutland@arm.com>
>> Cc: linux-arm-kernel@lists.infradead.org
>> Cc: linux-kernel@vger.kernel.org
>> Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
>> ---
> 
> [...]
> 
>>  void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
>>  {
>>  	struct brbe_hw_attr *brbe_attr = (struct brbe_hw_attr *)cpuc->percpu_pmu->private;
>> +	struct arm64_perf_task_context *task_ctx = event->pmu_ctx->task_ctx_data;
>> +	struct brbe_regset live[BRBE_MAX_ENTRIES];
>> +	int nr_live, nr_store;
>>  	u64 brbfcr, brbcr;
>> -	int idx, loop1_idx1, loop1_idx2, loop2_idx1, loop2_idx2, count;
>>  
>>  	brbcr = read_sysreg_s(SYS_BRBCR_EL1);
>>  	brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
>> @@ -739,36 +743,13 @@ void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
>>  	write_sysreg_s(brbfcr | BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
>>  	isb();
>>  
>> -	/* Determine the indices for each loop */
>> -	loop1_idx1 = BRBE_BANK0_IDX_MIN;
>> -	if (brbe_attr->brbe_nr <= BRBE_BANK_MAX_ENTRIES) {
>> -		loop1_idx2 = brbe_attr->brbe_nr - 1;
>> -		loop2_idx1 = BRBE_BANK1_IDX_MIN;
>> -		loop2_idx2 = BRBE_BANK0_IDX_MAX;
>> -	} else {
>> -		loop1_idx2 = BRBE_BANK0_IDX_MAX;
>> -		loop2_idx1 = BRBE_BANK1_IDX_MIN;
>> -		loop2_idx2 = brbe_attr->brbe_nr - 1;
>> -	}
>> -
>> -	/* Loop through bank 0 */
>> -	select_brbe_bank(BRBE_BANK_IDX_0);
>> -	for (idx = 0, count = loop1_idx1; count <= loop1_idx2; idx++, count++) {
>> -		if (!capture_branch_entry(cpuc, event, idx))
>> -			goto skip_bank_1;
>> -	}
>> -
>> -	/* Loop through bank 1 */
>> -	select_brbe_bank(BRBE_BANK_IDX_1);
>> -	for (count = loop2_idx1; count <= loop2_idx2; idx++, count++) {
>> -		if (!capture_branch_entry(cpuc, event, idx))
>> -			break;
>> -	}
>> -
>> -skip_bank_1:
>> -	cpuc->branches->branch_stack.nr = idx;
>> -	cpuc->branches->branch_stack.hw_idx = -1ULL;
>> +	nr_live = capture_brbe_regset(brbe_attr, live);
>> +	nr_store = task_ctx->nr_brbe_records;
>> +	nr_store = stitch_stored_live_entries(task_ctx->store, live, nr_store,
>> +					      nr_live, brbe_attr->brbe_nr);
>> +	process_branch_entries(cpuc, event, task_ctx->store, nr_store);
> 
> Hi Anshuman,
> 
> With the following command I get a crash:
> 
>   perf record --branch-filter any,save_type -a -- ls
> 
> [  101.171822] Unable to handle kernel NULL pointer dereference at
> virtual address 0000000000000600
> ...
> [145380.414654] Call trace:
> [145380.414739]  armv8pmu_branch_read+0x7c/0x578
> [145380.414895]  armv8pmu_handle_irq+0x104/0x1c0
> [145380.415043]  armpmu_dispatch_irq+0x38/0x70
> [145380.415209]  __handle_irq_event_percpu+0x124/0x3b8
> [145380.415392]  handle_irq_event+0x54/0xc8
> [145380.415567]  handle_fasteoi_irq+0x100/0x1e0
> [145380.415718]  generic_handle_domain_irq+0x38/0x58
> [145380.415895]  gic_handle_irq+0x5c/0x130
> [145380.416025]  call_on_irq_stack+0x24/0x58
> [145380.416173]  el1_interrupt+0x74/0xc0
> [145380.416321]  el1h_64_irq_handler+0x18/0x28
> [145380.416475]  el1h_64_irq+0x64/0x68
> [145380.416604]  smp_call_function_single+0xe8/0x1f0
> [145380.416745]  event_function_call+0xbc/0x1c8
> [145380.416919]  _perf_event_enable+0x84/0xa0
> [145380.417069]  perf_ioctl+0xe8/0xd68
> [145380.417204]  __arm64_sys_ioctl+0x9c/0xe0
> [145380.417353]  invoke_syscall+0x4c/0x120
> [145380.417523]  el0_svc_common+0xd0/0x120
> [145380.417693]  do_el0_svc+0x3c/0xb8
> [145380.417859]  el0_svc+0x50/0xc0
> [145380.418004]  el0t_64_sync_handler+0x84/0xf0
> [145380.418160]  el0t_64_sync+0x190/0x198
> 
> When using --branch-filter any,u without -a it seems to be fine so could
> be that task_ctx is null in per-cpu mode, or something to do with the
> userspace only flag?
> 
> I'm also wondering if it's possible to collapse some of the last 5
> commits? They seem to mostly modify things in brbe.c which is a new file
> so the history probably isn't important at this point it just makes it a
> bit harder to review.
> 

I realised I just tested V9 instead of V10 but I diffed them and don't
see anything that would change this issue so it's probably on both versions.

>>  	process_branch_aborts(cpuc);
>> +	task_ctx->nr_brbe_records = 0;
>>  
>>  	/* Unpause the buffer */
>>  	write_sysreg_s(brbfcr & ~BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
Anshuman Khandual May 24, 2023, 3:10 a.m. UTC | #3 
On 5/23/23 20:09, James Clark wrote:
> 
> 
> On 15/03/2023 05:14, Anshuman Khandual wrote:
>> This modifies armv8pmu_branch_read() to concatenate live entries along with
>> task context stored entries and then process the resultant buffer to create
>> perf branch entry array for perf_sample_data. It follows the same principle
>> like task sched out.
>>
>> Cc: Catalin Marinas <catalin.marinas@arm.com>
>> Cc: Will Deacon <will@kernel.org>
>> Cc: Mark Rutland <mark.rutland@arm.com>
>> Cc: linux-arm-kernel@lists.infradead.org
>> Cc: linux-kernel@vger.kernel.org
>> Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
>> ---
> 
> [...]
> 
>>  void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
>>  {
>>  	struct brbe_hw_attr *brbe_attr = (struct brbe_hw_attr *)cpuc->percpu_pmu->private;
>> +	struct arm64_perf_task_context *task_ctx = event->pmu_ctx->task_ctx_data;
>> +	struct brbe_regset live[BRBE_MAX_ENTRIES];
>> +	int nr_live, nr_store;
>>  	u64 brbfcr, brbcr;
>> -	int idx, loop1_idx1, loop1_idx2, loop2_idx1, loop2_idx2, count;
>>  
>>  	brbcr = read_sysreg_s(SYS_BRBCR_EL1);
>>  	brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
>> @@ -739,36 +743,13 @@ void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
>>  	write_sysreg_s(brbfcr | BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
>>  	isb();
>>  
>> -	/* Determine the indices for each loop */
>> -	loop1_idx1 = BRBE_BANK0_IDX_MIN;
>> -	if (brbe_attr->brbe_nr <= BRBE_BANK_MAX_ENTRIES) {
>> -		loop1_idx2 = brbe_attr->brbe_nr - 1;
>> -		loop2_idx1 = BRBE_BANK1_IDX_MIN;
>> -		loop2_idx2 = BRBE_BANK0_IDX_MAX;
>> -	} else {
>> -		loop1_idx2 = BRBE_BANK0_IDX_MAX;
>> -		loop2_idx1 = BRBE_BANK1_IDX_MIN;
>> -		loop2_idx2 = brbe_attr->brbe_nr - 1;
>> -	}
>> -
>> -	/* Loop through bank 0 */
>> -	select_brbe_bank(BRBE_BANK_IDX_0);
>> -	for (idx = 0, count = loop1_idx1; count <= loop1_idx2; idx++, count++) {
>> -		if (!capture_branch_entry(cpuc, event, idx))
>> -			goto skip_bank_1;
>> -	}
>> -
>> -	/* Loop through bank 1 */
>> -	select_brbe_bank(BRBE_BANK_IDX_1);
>> -	for (count = loop2_idx1; count <= loop2_idx2; idx++, count++) {
>> -		if (!capture_branch_entry(cpuc, event, idx))
>> -			break;
>> -	}
>> -
>> -skip_bank_1:
>> -	cpuc->branches->branch_stack.nr = idx;
>> -	cpuc->branches->branch_stack.hw_idx = -1ULL;
>> +	nr_live = capture_brbe_regset(brbe_attr, live);
>> +	nr_store = task_ctx->nr_brbe_records;
>> +	nr_store = stitch_stored_live_entries(task_ctx->store, live, nr_store,
>> +					      nr_live, brbe_attr->brbe_nr);
>> +	process_branch_entries(cpuc, event, task_ctx->store, nr_store);
> 
> Hi Anshuman,
> 
> With the following command I get a crash:
> 
>   perf record --branch-filter any,save_type -a -- ls
> 
> [  101.171822] Unable to handle kernel NULL pointer dereference at
> virtual address 0000000000000600
> ...
> [145380.414654] Call trace:
> [145380.414739]  armv8pmu_branch_read+0x7c/0x578
> [145380.414895]  armv8pmu_handle_irq+0x104/0x1c0
> [145380.415043]  armpmu_dispatch_irq+0x38/0x70
> [145380.415209]  __handle_irq_event_percpu+0x124/0x3b8
> [145380.415392]  handle_irq_event+0x54/0xc8
> [145380.415567]  handle_fasteoi_irq+0x100/0x1e0
> [145380.415718]  generic_handle_domain_irq+0x38/0x58
> [145380.415895]  gic_handle_irq+0x5c/0x130
> [145380.416025]  call_on_irq_stack+0x24/0x58
> [145380.416173]  el1_interrupt+0x74/0xc0
> [145380.416321]  el1h_64_irq_handler+0x18/0x28
> [145380.416475]  el1h_64_irq+0x64/0x68
> [145380.416604]  smp_call_function_single+0xe8/0x1f0
> [145380.416745]  event_function_call+0xbc/0x1c8
> [145380.416919]  _perf_event_enable+0x84/0xa0
> [145380.417069]  perf_ioctl+0xe8/0xd68
> [145380.417204]  __arm64_sys_ioctl+0x9c/0xe0
> [145380.417353]  invoke_syscall+0x4c/0x120
> [145380.417523]  el0_svc_common+0xd0/0x120
> [145380.417693]  do_el0_svc+0x3c/0xb8
> [145380.417859]  el0_svc+0x50/0xc0
> [145380.418004]  el0t_64_sync_handler+0x84/0xf0
> [145380.418160]  el0t_64_sync+0x190/0x198
> 
> When using --branch-filter any,u without -a it seems to be fine so could
> be that task_ctx is null in per-cpu mode, or something to do with the
> userspace only flag?

It's task_ctx is null in per-cpu mode, because armv8pmu_branch_read() used
event->pmu_ctx->task_ctx_data in per-cpu mode where it would not have been
allocated as well. Following change fixes the problem.

diff --git a/drivers/perf/arm_brbe.c b/drivers/perf/arm_brbe.c
index 9e441141a2c3..c8fd581eacf9 100644
--- a/drivers/perf/arm_brbe.c
+++ b/drivers/perf/arm_brbe.c
@@ -744,12 +744,17 @@ void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
        isb();
 
        nr_live = capture_brbe_regset(brbe_attr, live);
-       nr_store = task_ctx->nr_brbe_records;
-       nr_store = stitch_stored_live_entries(task_ctx->store, live, nr_store,
-                                             nr_live, brbe_attr->brbe_nr);
-       process_branch_entries(cpuc, event, task_ctx->store, nr_store);
+       if (event->ctx->task) {
+               nr_store = task_ctx->nr_brbe_records;
+               nr_store = stitch_stored_live_entries(task_ctx->store, live, nr_store,
+                                                     nr_live, brbe_attr->brbe_nr);
+               process_branch_entries(cpuc, event, task_ctx->store, nr_store);
+               task_ctx->nr_brbe_records = 0;
+       } else {
+               process_branch_entries(cpuc, event, live, nr_live);
+       }
+
        process_branch_aborts(cpuc);
-       task_ctx->nr_brbe_records = 0;
 
        /* Unpause the buffer */
        write_sysreg_s(brbfcr & ~BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);


> 
> I'm also wondering if it's possible to collapse some of the last 5
> commits? They seem to mostly modify things in brbe.c which is a new file
> so the history probably isn't important at this point it just makes it a
> bit harder to review.

[PATCH 6/10] enables base perf branch stack sampling on arm64 platform via
BRBE and then subsequent patches represent logical progression up until
save-stitch mechanism is implemented both during normal PMU IRQ and task
switch callbacks.

> 
>>  	process_branch_aborts(cpuc);
>> +	task_ctx->nr_brbe_records = 0;
>>  
>>  	/* Unpause the buffer */
>>  	write_sysreg_s(brbfcr & ~BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
diff mbox series

Patch

diff --git a/arch/arm64/kernel/brbe.c b/arch/arm64/kernel/brbe.c
index 3dcb4407b92a..652af6668d37 100644
--- a/arch/arm64/kernel/brbe.c
+++ b/arch/arm64/kernel/brbe.c
@@ -693,41 +693,45 @@  void armv8pmu_branch_reset(void)
 	isb();
 }
 
-static bool capture_branch_entry(struct pmu_hw_events *cpuc,
-				 struct perf_event *event, int idx)
+static void brbe_regset_branch_entries(struct pmu_hw_events *cpuc, struct perf_event *event,
+				       struct brbe_regset *regset, int idx)
 {
 	struct perf_branch_entry *entry = &cpuc->branches->branch_entries[idx];
-	u64 brbinf = get_brbinf_reg(idx);
-
-	/*
-	 * There are no valid entries anymore on the buffer.
-	 * Abort the branch record processing to save some
-	 * cycles and also reduce the capture/process load
-	 * for the user space as well.
-	 */
-	if (brbe_invalid(brbinf))
-		return false;
+	u64 brbinf = regset[idx].brbinf;
 
 	perf_clear_branch_entry_bitfields(entry);
 	if (brbe_record_is_complete(brbinf)) {
-		entry->from = get_brbsrc_reg(idx);
-		entry->to = get_brbtgt_reg(idx);
+		entry->from = regset[idx].brbsrc;
+		entry->to = regset[idx].brbtgt;
 	} else if (brbe_record_is_source_only(brbinf)) {
-		entry->from = get_brbsrc_reg(idx);
+		entry->from = regset[idx].brbsrc;
 		entry->to = 0;
 	} else if (brbe_record_is_target_only(brbinf)) {
 		entry->from = 0;
-		entry->to = get_brbtgt_reg(idx);
+		entry->to = regset[idx].brbtgt;
 	}
 	capture_brbe_flags(entry, event, brbinf);
-	return true;
+}
+
+static void process_branch_entries(struct pmu_hw_events *cpuc, struct perf_event *event,
+				   struct brbe_regset *regset, int nr_regset)
+{
+	int idx;
+
+	for (idx = 0; idx < nr_regset; idx++)
+		brbe_regset_branch_entries(cpuc, event, regset, idx);
+
+	cpuc->branches->branch_stack.nr = nr_regset;
+	cpuc->branches->branch_stack.hw_idx = -1ULL;
 }
 
 void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
 {
 	struct brbe_hw_attr *brbe_attr = (struct brbe_hw_attr *)cpuc->percpu_pmu->private;
+	struct arm64_perf_task_context *task_ctx = event->pmu_ctx->task_ctx_data;
+	struct brbe_regset live[BRBE_MAX_ENTRIES];
+	int nr_live, nr_store;
 	u64 brbfcr, brbcr;
-	int idx, loop1_idx1, loop1_idx2, loop2_idx1, loop2_idx2, count;
 
 	brbcr = read_sysreg_s(SYS_BRBCR_EL1);
 	brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
@@ -739,36 +743,13 @@  void armv8pmu_branch_read(struct pmu_hw_events *cpuc, struct perf_event *event)
 	write_sysreg_s(brbfcr | BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);
 	isb();
 
-	/* Determine the indices for each loop */
-	loop1_idx1 = BRBE_BANK0_IDX_MIN;
-	if (brbe_attr->brbe_nr <= BRBE_BANK_MAX_ENTRIES) {
-		loop1_idx2 = brbe_attr->brbe_nr - 1;
-		loop2_idx1 = BRBE_BANK1_IDX_MIN;
-		loop2_idx2 = BRBE_BANK0_IDX_MAX;
-	} else {
-		loop1_idx2 = BRBE_BANK0_IDX_MAX;
-		loop2_idx1 = BRBE_BANK1_IDX_MIN;
-		loop2_idx2 = brbe_attr->brbe_nr - 1;
-	}
-
-	/* Loop through bank 0 */
-	select_brbe_bank(BRBE_BANK_IDX_0);
-	for (idx = 0, count = loop1_idx1; count <= loop1_idx2; idx++, count++) {
-		if (!capture_branch_entry(cpuc, event, idx))
-			goto skip_bank_1;
-	}
-
-	/* Loop through bank 1 */
-	select_brbe_bank(BRBE_BANK_IDX_1);
-	for (count = loop2_idx1; count <= loop2_idx2; idx++, count++) {
-		if (!capture_branch_entry(cpuc, event, idx))
-			break;
-	}
-
-skip_bank_1:
-	cpuc->branches->branch_stack.nr = idx;
-	cpuc->branches->branch_stack.hw_idx = -1ULL;
+	nr_live = capture_brbe_regset(brbe_attr, live);
+	nr_store = task_ctx->nr_brbe_records;
+	nr_store = stitch_stored_live_entries(task_ctx->store, live, nr_store,
+					      nr_live, brbe_attr->brbe_nr);
+	process_branch_entries(cpuc, event, task_ctx->store, nr_store);
 	process_branch_aborts(cpuc);
+	task_ctx->nr_brbe_records = 0;
 
 	/* Unpause the buffer */
 	write_sysreg_s(brbfcr & ~BRBFCR_EL1_PAUSED, SYS_BRBFCR_EL1);