[4/7] perf record: Record dropped sample count
Commit Message
When it uses bpf filters, event might drop some samples. It'd be nice
if it can report how many samples it lost. As LOST_SAMPLES event can
carry the similar information, let's use it for bpf filters.
To indicate it's from BPF filters, add a new misc flag for that and
do not display cpu load warnings.
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
tools/perf/builtin-record.c | 37 ++++++++++++++++++++++--------------
tools/perf/util/bpf-filter.c | 7 +++++++
tools/perf/util/bpf-filter.h | 5 +++++
tools/perf/util/session.c | 3 ++-
4 files changed, 37 insertions(+), 15 deletions(-)
Comments
On Mon, Feb 13, 2023 at 9:05 PM Namhyung Kim <namhyung@kernel.org> wrote:
>
> When it uses bpf filters, event might drop some samples. It'd be nice
> if it can report how many samples it lost. As LOST_SAMPLES event can
> carry the similar information, let's use it for bpf filters.
>
> To indicate it's from BPF filters, add a new misc flag for that and
> do not display cpu load warnings.
Can you potentially have lost samples from being too slow to drain the
ring buffer and dropped samples because of BPF? Is it possible to
distinguish lost and dropped with this approach?
Thanks,
Ian
> Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> ---
> tools/perf/builtin-record.c | 37 ++++++++++++++++++++++--------------
> tools/perf/util/bpf-filter.c | 7 +++++++
> tools/perf/util/bpf-filter.h | 5 +++++
> tools/perf/util/session.c | 3 ++-
> 4 files changed, 37 insertions(+), 15 deletions(-)
>
> diff --git a/tools/perf/builtin-record.c b/tools/perf/builtin-record.c
> index c81047a78f3e..3201d1a1ea1f 100644
> --- a/tools/perf/builtin-record.c
> +++ b/tools/perf/builtin-record.c
> @@ -1869,24 +1869,16 @@ record__switch_output(struct record *rec, bool at_exit)
> return fd;
> }
>
> -static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
> +static void __record__save_lost_samples(struct record *rec, struct evsel *evsel,
> struct perf_record_lost_samples *lost,
> - int cpu_idx, int thread_idx)
> + int cpu_idx, int thread_idx, u64 lost_count,
> + u16 misc_flag)
> {
> - struct perf_counts_values count;
> struct perf_sample_id *sid;
> struct perf_sample sample = {};
> int id_hdr_size;
>
> - if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
> - pr_err("read LOST count failed\n");
> - return;
> - }
> -
> - if (count.lost == 0)
> - return;
> -
> - lost->lost = count.lost;
> + lost->lost = lost_count;
> if (evsel->core.ids) {
> sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
> sample.id = sid->id;
> @@ -1895,6 +1887,7 @@ static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
> id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
> evsel->core.attr.sample_type, &sample);
> lost->header.size = sizeof(*lost) + id_hdr_size;
> + lost->header.misc = misc_flag;
> record__write(rec, NULL, lost, lost->header.size);
> }
>
> @@ -1918,6 +1911,7 @@ static void record__read_lost_samples(struct record *rec)
>
> evlist__for_each_entry(session->evlist, evsel) {
> struct xyarray *xy = evsel->core.sample_id;
> + u64 lost_count;
>
> if (xy == NULL || evsel->core.fd == NULL)
> continue;
> @@ -1929,12 +1923,27 @@ static void record__read_lost_samples(struct record *rec)
>
> for (int x = 0; x < xyarray__max_x(xy); x++) {
> for (int y = 0; y < xyarray__max_y(xy); y++) {
> - __record__read_lost_samples(rec, evsel, lost, x, y);
> + struct perf_counts_values count;
> +
> + if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
> + pr_err("read LOST count failed\n");
> + goto out;
> + }
> +
> + if (count.lost) {
> + __record__save_lost_samples(rec, evsel, lost,
> + x, y, count.lost, 0);
> + }
> }
> }
> +
> + lost_count = perf_bpf_filter__lost_count(evsel);
> + if (lost_count)
> + __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
> + PERF_RECORD_MISC_LOST_SAMPLES_BPF);
> }
> +out:
> free(lost);
> -
> }
>
> static volatile sig_atomic_t workload_exec_errno;
> diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c
> index f47420cf81c9..11fb391c92e9 100644
> --- a/tools/perf/util/bpf-filter.c
> +++ b/tools/perf/util/bpf-filter.c
> @@ -76,6 +76,13 @@ int perf_bpf_filter__destroy(struct evsel *evsel)
> return 0;
> }
>
> +u64 perf_bpf_filter__lost_count(struct evsel *evsel)
> +{
> + struct sample_filter_bpf *skel = evsel->bpf_skel;
> +
> + return skel ? skel->bss->dropped : 0;
> +}
> +
> struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flags,
> enum perf_bpf_filter_op op,
> unsigned long val)
> diff --git a/tools/perf/util/bpf-filter.h b/tools/perf/util/bpf-filter.h
> index 6077930073f9..36b44c8188ab 100644
> --- a/tools/perf/util/bpf-filter.h
> +++ b/tools/perf/util/bpf-filter.h
> @@ -22,6 +22,7 @@ struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flag
> int perf_bpf_filter__parse(struct list_head *expr_head, const char *str);
> int perf_bpf_filter__prepare(struct evsel *evsel);
> int perf_bpf_filter__destroy(struct evsel *evsel);
> +u64 perf_bpf_filter__lost_count(struct evsel *evsel);
>
> #else /* !HAVE_BPF_SKEL */
>
> @@ -38,5 +39,9 @@ static inline int perf_bpf_filter__destroy(struct evsel *evsel)
> {
> return -ENOSYS;
> }
> +static inline u64 perf_bpf_filter__lost_count(struct evsel *evsel)
> +{
> + return 0;
> +}
> #endif /* HAVE_BPF_SKEL*/
> #endif /* PERF_UTIL_BPF_FILTER_H */
> \ No newline at end of file
> diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c
> index 749d5b5c135b..7d8d057d1772 100644
> --- a/tools/perf/util/session.c
> +++ b/tools/perf/util/session.c
> @@ -1582,7 +1582,8 @@ static int machines__deliver_event(struct machines *machines,
> evlist->stats.total_lost += event->lost.lost;
> return tool->lost(tool, event, sample, machine);
> case PERF_RECORD_LOST_SAMPLES:
> - if (tool->lost_samples == perf_event__process_lost_samples)
> + if (tool->lost_samples == perf_event__process_lost_samples &&
> + !(event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF))
> evlist->stats.total_lost_samples += event->lost_samples.lost;
> return tool->lost_samples(tool, event, sample, machine);
> case PERF_RECORD_READ:
> --
> 2.39.1.581.gbfd45094c4-goog
>
On Tue, Feb 14, 2023 at 8:41 AM Ian Rogers <irogers@google.com> wrote:
>
> On Mon, Feb 13, 2023 at 9:05 PM Namhyung Kim <namhyung@kernel.org> wrote:
> >
> > When it uses bpf filters, event might drop some samples. It'd be nice
> > if it can report how many samples it lost. As LOST_SAMPLES event can
> > carry the similar information, let's use it for bpf filters.
> >
> > To indicate it's from BPF filters, add a new misc flag for that and
> > do not display cpu load warnings.
>
> Can you potentially have lost samples from being too slow to drain the
> ring buffer and dropped samples because of BPF? Is it possible to
> distinguish lost and dropped with this approach?
Yeah, the former is exactly what LOST_SAMPLES event gives you.
It should come from the kernel while BPF filters keep a separate
counter for dropped samples and inject LOST_SAMPLES events
with the new misc flag. So we can differentiate them using the misc
flag and that's how I suppress the warning for BPF dropped ones.
Thanks,
Namhyung
On Mon, Feb 13, 2023 at 09:04:49PM -0800, Namhyung Kim wrote:
SNIP
> @@ -1929,12 +1923,27 @@ static void record__read_lost_samples(struct record *rec)
>
> for (int x = 0; x < xyarray__max_x(xy); x++) {
> for (int y = 0; y < xyarray__max_y(xy); y++) {
> - __record__read_lost_samples(rec, evsel, lost, x, y);
> + struct perf_counts_values count;
> +
> + if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
> + pr_err("read LOST count failed\n");
> + goto out;
> + }
> +
> + if (count.lost) {
> + __record__save_lost_samples(rec, evsel, lost,
> + x, y, count.lost, 0);
> + }
> }
> }
> +
> + lost_count = perf_bpf_filter__lost_count(evsel);
> + if (lost_count)
> + __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
> + PERF_RECORD_MISC_LOST_SAMPLES_BPF);
hi,
I can't see PERF_RECORD_MISC_LOST_SAMPLES_BPF in the tip/perf/core so can't compile,
what do I miss?
thanks,
jirka
Em Thu, Feb 16, 2023 at 05:23:05PM +0100, Jiri Olsa escreveu:
> On Mon, Feb 13, 2023 at 09:04:49PM -0800, Namhyung Kim wrote:
>
> SNIP
>
> > @@ -1929,12 +1923,27 @@ static void record__read_lost_samples(struct record *rec)
> >
> > for (int x = 0; x < xyarray__max_x(xy); x++) {
> > for (int y = 0; y < xyarray__max_y(xy); y++) {
> > - __record__read_lost_samples(rec, evsel, lost, x, y);
> > + struct perf_counts_values count;
> > +
> > + if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
> > + pr_err("read LOST count failed\n");
> > + goto out;
> > + }
> > +
> > + if (count.lost) {
> > + __record__save_lost_samples(rec, evsel, lost,
> > + x, y, count.lost, 0);
> > + }
> > }
> > }
> > +
> > + lost_count = perf_bpf_filter__lost_count(evsel);
> > + if (lost_count)
> > + __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
> > + PERF_RECORD_MISC_LOST_SAMPLES_BPF);
>
> hi,
> I can't see PERF_RECORD_MISC_LOST_SAMPLES_BPF in the tip/perf/core so can't compile,
> what do I miss?
Humm, but you shouldn't need kernel headers to build tools/perf/, right?
- Arnaldo
On Thu, Feb 16, 2023 at 01:32:14PM -0300, Arnaldo Carvalho de Melo wrote:
> Em Thu, Feb 16, 2023 at 05:23:05PM +0100, Jiri Olsa escreveu:
> > On Mon, Feb 13, 2023 at 09:04:49PM -0800, Namhyung Kim wrote:
> >
> > SNIP
> >
> > > @@ -1929,12 +1923,27 @@ static void record__read_lost_samples(struct record *rec)
> > >
> > > for (int x = 0; x < xyarray__max_x(xy); x++) {
> > > for (int y = 0; y < xyarray__max_y(xy); y++) {
> > > - __record__read_lost_samples(rec, evsel, lost, x, y);
> > > + struct perf_counts_values count;
> > > +
> > > + if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
> > > + pr_err("read LOST count failed\n");
> > > + goto out;
> > > + }
> > > +
> > > + if (count.lost) {
> > > + __record__save_lost_samples(rec, evsel, lost,
> > > + x, y, count.lost, 0);
> > > + }
> > > }
> > > }
> > > +
> > > + lost_count = perf_bpf_filter__lost_count(evsel);
> > > + if (lost_count)
> > > + __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
> > > + PERF_RECORD_MISC_LOST_SAMPLES_BPF);
> >
> > hi,
> > I can't see PERF_RECORD_MISC_LOST_SAMPLES_BPF in the tip/perf/core so can't compile,
> > what do I miss?
>
> Humm, but you shouldn't need kernel headers to build tools/perf/, right?
right, should be also in tools/include headers
jirka
Hi Jiri and Arnaldo,
On Thu, Feb 16, 2023 at 05:34:33PM +0100, Jiri Olsa wrote:
> On Thu, Feb 16, 2023 at 01:32:14PM -0300, Arnaldo Carvalho de Melo wrote:
> > Em Thu, Feb 16, 2023 at 05:23:05PM +0100, Jiri Olsa escreveu:
> > > On Mon, Feb 13, 2023 at 09:04:49PM -0800, Namhyung Kim wrote:
> > >
> > > SNIP
> > >
> > > > @@ -1929,12 +1923,27 @@ static void record__read_lost_samples(struct record *rec)
> > > >
> > > > for (int x = 0; x < xyarray__max_x(xy); x++) {
> > > > for (int y = 0; y < xyarray__max_y(xy); y++) {
> > > > - __record__read_lost_samples(rec, evsel, lost, x, y);
> > > > + struct perf_counts_values count;
> > > > +
> > > > + if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
> > > > + pr_err("read LOST count failed\n");
> > > > + goto out;
> > > > + }
> > > > +
> > > > + if (count.lost) {
> > > > + __record__save_lost_samples(rec, evsel, lost,
> > > > + x, y, count.lost, 0);
> > > > + }
> > > > }
> > > > }
> > > > +
> > > > + lost_count = perf_bpf_filter__lost_count(evsel);
> > > > + if (lost_count)
> > > > + __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
> > > > + PERF_RECORD_MISC_LOST_SAMPLES_BPF);
> > >
> > > hi,
> > > I can't see PERF_RECORD_MISC_LOST_SAMPLES_BPF in the tip/perf/core so can't compile,
> > > what do I miss?
> >
> > Humm, but you shouldn't need kernel headers to build tools/perf/, right?
>
> right, should be also in tools/include headers
Yeah, sorry about that. I'm not sure how I missed the part.
I put it in tools/lib/perf/include/perf/event.h only as it does nothing
with kernel. Will fix in v2.
Thanks,
Namhyung
---8<---
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index ad47d7b31046..51b9338f4c11 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -70,6 +70,8 @@ struct perf_record_lost {
__u64 lost;
};
+#define PERF_RECORD_MISC_LOST_SAMPLES_BPF (1 << 15)
+
struct perf_record_lost_samples {
struct perf_event_header header;
__u64 lost;
@@ -1869,24 +1869,16 @@ record__switch_output(struct record *rec, bool at_exit)
return fd;
}
-static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
+static void __record__save_lost_samples(struct record *rec, struct evsel *evsel,
struct perf_record_lost_samples *lost,
- int cpu_idx, int thread_idx)
+ int cpu_idx, int thread_idx, u64 lost_count,
+ u16 misc_flag)
{
- struct perf_counts_values count;
struct perf_sample_id *sid;
struct perf_sample sample = {};
int id_hdr_size;
- if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
- pr_err("read LOST count failed\n");
- return;
- }
-
- if (count.lost == 0)
- return;
-
- lost->lost = count.lost;
+ lost->lost = lost_count;
if (evsel->core.ids) {
sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
sample.id = sid->id;
@@ -1895,6 +1887,7 @@ static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
evsel->core.attr.sample_type, &sample);
lost->header.size = sizeof(*lost) + id_hdr_size;
+ lost->header.misc = misc_flag;
record__write(rec, NULL, lost, lost->header.size);
}
@@ -1918,6 +1911,7 @@ static void record__read_lost_samples(struct record *rec)
evlist__for_each_entry(session->evlist, evsel) {
struct xyarray *xy = evsel->core.sample_id;
+ u64 lost_count;
if (xy == NULL || evsel->core.fd == NULL)
continue;
@@ -1929,12 +1923,27 @@ static void record__read_lost_samples(struct record *rec)
for (int x = 0; x < xyarray__max_x(xy); x++) {
for (int y = 0; y < xyarray__max_y(xy); y++) {
- __record__read_lost_samples(rec, evsel, lost, x, y);
+ struct perf_counts_values count;
+
+ if (perf_evsel__read(&evsel->core, x, y, &count) < 0) {
+ pr_err("read LOST count failed\n");
+ goto out;
+ }
+
+ if (count.lost) {
+ __record__save_lost_samples(rec, evsel, lost,
+ x, y, count.lost, 0);
+ }
}
}
+
+ lost_count = perf_bpf_filter__lost_count(evsel);
+ if (lost_count)
+ __record__save_lost_samples(rec, evsel, lost, 0, 0, lost_count,
+ PERF_RECORD_MISC_LOST_SAMPLES_BPF);
}
+out:
free(lost);
-
}
static volatile sig_atomic_t workload_exec_errno;
@@ -76,6 +76,13 @@ int perf_bpf_filter__destroy(struct evsel *evsel)
return 0;
}
+u64 perf_bpf_filter__lost_count(struct evsel *evsel)
+{
+ struct sample_filter_bpf *skel = evsel->bpf_skel;
+
+ return skel ? skel->bss->dropped : 0;
+}
+
struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flags,
enum perf_bpf_filter_op op,
unsigned long val)
@@ -22,6 +22,7 @@ struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(unsigned long sample_flag
int perf_bpf_filter__parse(struct list_head *expr_head, const char *str);
int perf_bpf_filter__prepare(struct evsel *evsel);
int perf_bpf_filter__destroy(struct evsel *evsel);
+u64 perf_bpf_filter__lost_count(struct evsel *evsel);
#else /* !HAVE_BPF_SKEL */
@@ -38,5 +39,9 @@ static inline int perf_bpf_filter__destroy(struct evsel *evsel)
{
return -ENOSYS;
}
+static inline u64 perf_bpf_filter__lost_count(struct evsel *evsel)
+{
+ return 0;
+}
#endif /* HAVE_BPF_SKEL*/
#endif /* PERF_UTIL_BPF_FILTER_H */
\ No newline at end of file
@@ -1582,7 +1582,8 @@ static int machines__deliver_event(struct machines *machines,
evlist->stats.total_lost += event->lost.lost;
return tool->lost(tool, event, sample, machine);
case PERF_RECORD_LOST_SAMPLES:
- if (tool->lost_samples == perf_event__process_lost_samples)
+ if (tool->lost_samples == perf_event__process_lost_samples &&
+ !(event->header.misc & PERF_RECORD_MISC_LOST_SAMPLES_BPF))
evlist->stats.total_lost_samples += event->lost_samples.lost;
return tool->lost_samples(tool, event, sample, machine);
case PERF_RECORD_READ: