From 5c33a80edb016c2241cebabc07fc42698ee5d513 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 1 Jul 2026 12:49:43 -0400 Subject: [PATCH] call_rcu: pin per-CPU worker at thread startup create_all_cpu_call_rcu_data() gives each reclaim worker a target CPU (crdp->cpu_affinity), but the worker only pins to it lazily. Both the startup call and the in-loop re-check go through set_thread_cpu_affinity(), which rate-limits itself with if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK) return 0; gp_count starts at 0, so the first call returns without pinning; the pin only fires once gp_count reaches a multiple of SET_AFFINITY_CHECK_PERIOD (256). set_thread_cpu_affinity() is called once per work-loop iteration and each iteration sleeps at least 10 ms (poll), so the first pin does not happen until roughly 2.5 s into the thread's life. Until then the worker keeps its inherited (wide) affinity mask and the scheduler may place it on a different last-level-cache domain than the CPU whose callbacks it services: every freed object's cache line then bounces across the LLC boundary and the wake-up futex crosses it too. On a 192-core / 24-CCX machine this shows up as bimodal single-writer reclaim throughput (worker same-CCX vs cross-CCX) and depressed, high-variance throughput at scale. Split the actual pinning into do_set_thread_cpu_affinity() and call it unconditionally once at call_rcu thread startup, before processing any callback. set_thread_cpu_affinity() keeps the rate-limited migration-recovery re-check in the work loop. This only makes the already-intended placement happen promptly instead of ~2.5 s late. Measured (churn micro-benchmark, per-CPU call_rcu, EPYC 9654): single-updater reclaim goes from a bimodal ~2-6 Mops/s to a stable ~7.8; a multi-writer coherent-list workload improves +29% at 64 writers and +53% at 191 writers, with the run-to-run variance removed. Signed-off-by: Mathieu Desnoyers Change-Id: I76230a964126d14ad6ebd7fcd6783f4238b5ae85 --- src/urcu-call-rcu-impl.h | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h index e473f4a93..6db4ec015 100644 --- a/src/urcu-call-rcu-impl.h +++ b/src/urcu-call-rcu-impl.h @@ -190,18 +190,20 @@ static void call_rcu_unlock(pthread_mutex_t *pmp) * cpuset(7). */ #ifdef HAVE_SCHED_SETAFFINITY +/* + * Actually pin the calling thread to crdp->cpu_affinity. Unconditional + * (no rate-limit), so it is used for the initial placement at call_rcu + * thread startup; the work loop reaches it through the rate-limited + * set_thread_cpu_affinity() wrapper below. + */ static -int set_thread_cpu_affinity(struct call_rcu_data *crdp) +int do_set_thread_cpu_affinity(struct call_rcu_data *crdp) { cpu_set_t mask; int ret; if (crdp->cpu_affinity < 0) return 0; - if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK) - return 0; - if (urcu_sched_getcpu() == crdp->cpu_affinity) - return 0; CPU_ZERO(&mask); CPU_SET(crdp->cpu_affinity, &mask); @@ -218,8 +220,26 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp) } return ret; } + +static +int set_thread_cpu_affinity(struct call_rcu_data *crdp) +{ + if (crdp->cpu_affinity < 0) + return 0; + /* Rate-limit the migration-recovery re-pin in the work loop. */ + if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK) + return 0; + if (urcu_sched_getcpu() == crdp->cpu_affinity) + return 0; + return do_set_thread_cpu_affinity(crdp); +} #else static +int do_set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((__unused__))) +{ + return 0; +} +static int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((__unused__))) { return 0; @@ -318,7 +338,8 @@ static void *call_rcu_thread(void *arg) struct call_rcu_data *crdp = (struct call_rcu_data *) arg; int rt = !!(uatomic_load(&crdp->flags) & URCU_CALL_RCU_RT); - if (set_thread_cpu_affinity(crdp)) + /* Initial placement: pin immediately, before processing any callback. */ + if (do_set_thread_cpu_affinity(crdp)) urcu_die(errno); /*