From 7eb03ba629b3205696efaefaa51dd37b519ac128 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Thu, 9 Jul 2026 15:48:33 -0400 Subject: [PATCH] call-rcu worker: set CPU affinity on first non-empty dequeue commit 28f282af1905 ("call_rcu: pin per-CPU worker at thread startup") attempts to prevent the per-cpu call-rcu worker from running on the wrong CPU at the beginning of the application lifetime (first ~2.5s). The goal is to ensure good performance in that time-frame. However, pinning each worker threads at thread startup for each CPU for per-cpu call-rcu worker configurations will trigger a lot of CPU wakeups on short-lived processes using few CPUs when they are deployed on large SMP machines: this wakes up every CPU just to set the affinity. This is detrimental for energy consumption of the CPUs. Refactor the call-rcu worker thread code to call set_thread_cpu_affinity() only when call-rcu work is dequeued, and introduce an "affinity_init" boolean which allows the first call to enforce affinity initialization, bypassing the period mask check and the current CPU vs affinity CPU check. Signed-off-by: Mathieu Desnoyers Change-Id: I24ee28b5133cbebd33af076321ba18aaab4d0b8e --- src/urcu-call-rcu-impl.h | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h index 0dfa53fe..dc253d7c 100644 --- a/src/urcu-call-rcu-impl.h +++ b/src/urcu-call-rcu-impl.h @@ -53,6 +53,7 @@ struct call_rcu_data { pthread_t tid; int cpu_affinity; unsigned long gp_count; + bool affinity_init; struct cds_list_head list; } __attribute__((__aligned__(CAA_CACHE_LINE_SIZE))); @@ -190,12 +191,6 @@ static void call_rcu_unlock(pthread_mutex_t *pmp) * cpuset(7). */ #ifdef HAVE_SCHED_SETAFFINITY -/* - * Actually pin the calling thread to crdp->cpu_affinity. Unconditional - * (no rate-limit), so it is used for the initial placement at call_rcu - * thread startup; the work loop reaches it through the rate-limited - * set_thread_cpu_affinity() wrapper below. - */ static int do_set_thread_cpu_affinity(struct call_rcu_data *crdp) { @@ -226,11 +221,18 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp) { if (crdp->cpu_affinity < 0) return 0; - /* Rate-limit the migration-recovery re-pin in the work loop. */ - if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK) - return 0; - if (urcu_sched_getcpu() == crdp->cpu_affinity) - return 0; + crdp->gp_count++; + /* Force setting the affinity if it was not already initialized. */ + if (!crdp->affinity_init) { + crdp->affinity_init = true; + } else { + /* Rate-limit the migration-recovery re-pin in the work loop. */ + if (crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK) + return 0; + /* Only set affinity if we happen to be running on the wrong CPU. */ + if (urcu_sched_getcpu() == crdp->cpu_affinity) + return 0; + } return do_set_thread_cpu_affinity(crdp); } #else @@ -338,10 +340,6 @@ static void *call_rcu_thread(void *arg) struct call_rcu_data *crdp = (struct call_rcu_data *) arg; int rt = !!(uatomic_load(&crdp->flags) & URCU_CALL_RCU_RT); - /* Initial placement: pin immediately, before processing any callback. */ - if (do_set_thread_cpu_affinity(crdp)) - urcu_die(errno); - /* * If callbacks take a read-side lock, we need to be registered. */ @@ -359,9 +357,6 @@ static void *call_rcu_thread(void *arg) struct cds_wfcq_node *cbs, *cbs_tmp_n; enum cds_wfcq_ret splice_ret; - if (set_thread_cpu_affinity(crdp)) - urcu_die(errno); - if (uatomic_load(&crdp->flags) & URCU_CALL_RCU_PAUSE) { /* * Pause requested. Become quiescent: remove @@ -385,6 +380,8 @@ static void *call_rcu_thread(void *arg) urcu_posix_assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK); urcu_posix_assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY); if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) { + if (set_thread_cpu_affinity(crdp)) + urcu_die(errno); synchronize_rcu(); cbcount = 0; __cds_wfcq_for_each_blocking_safe(&cbs_tmp_head, @@ -457,6 +454,7 @@ static void call_rcu_data_init(struct call_rcu_data **crdpp, cds_list_add(&crdp->list, &call_rcu_data_list); crdp->cpu_affinity = cpu_affinity; crdp->gp_count = 0; + crdp->affinity_init = false; rcu_set_pointer(crdpp, crdp); ret = sigfillset(&newmask);