From: Duncan Eastoe <duncan.eastoe@att.com>
Date: Fri, 18 Dec 2020 14:04:45 +0000 (+0000)
Subject: zebra: reduce dplane_fpm_nl ctxqueue_mutex contention
X-Git-Tag: base_7.6~109^2~3
X-Git-Url: https://git.puffer.fish/?a=commitdiff_plain;h=dc693fe057ce2c42534aa9336bc50ed24a182877;p=matthieu%2Ffrr.git

zebra: reduce dplane_fpm_nl ctxqueue_mutex contention

Reduce code in the critical sections of fpm_nl_process() and
fpm_process_queue() to the bare minimum - basically only enqueue
and dequeue operations on the shared ctxqueue.

Signed-off-by: Duncan Eastoe <duncan.eastoe@att.com>
---

diff --git a/zebra/dplane_fpm_nl.c b/zebra/dplane_fpm_nl.c
index 261b859bf6..bd7b604bad 100644
--- a/zebra/dplane_fpm_nl.c
+++ b/zebra/dplane_fpm_nl.c
@@ -1223,15 +1223,15 @@ static int fpm_process_queue(struct thread *t)
 	struct fpm_nl_ctx *fnc = THREAD_ARG(t);
 	struct zebra_dplane_ctx *ctx;
 
-	frr_mutex_lock_autounlock(&fnc->ctxqueue_mutex);
-
 	while (true) {
 		/* No space available yet. */
 		if (STREAM_WRITEABLE(fnc->obuf) < NL_PKT_BUF_SIZE)
 			break;
 
 		/* Dequeue next item or quit processing. */
-		ctx = dplane_ctx_dequeue(&fnc->ctxqueue);
+		frr_with_mutex (&fnc->ctxqueue_mutex) {
+			ctx = dplane_ctx_dequeue(&fnc->ctxqueue);
+		}
 		if (ctx == NULL)
 			break;
 
@@ -1435,12 +1435,17 @@ static int fpm_nl_process(struct zebra_dplane_provider *prov)
 		 * anyway.
 		 */
 		if (fnc->socket != -1 && fnc->connecting == false) {
-			frr_mutex_lock_autounlock(&fnc->ctxqueue_mutex);
-			dplane_ctx_enqueue_tail(&fnc->ctxqueue, ctx);
-
-			/* Account the number of contexts. */
+			/*
+			 * Update the number of queued contexts *before*
+			 * enqueueing, to ensure counter consistency.
+			 */
 			atomic_fetch_add_explicit(&fnc->counters.ctxqueue_len,
 						  1, memory_order_relaxed);
+
+			frr_with_mutex (&fnc->ctxqueue_mutex) {
+				dplane_ctx_enqueue_tail(&fnc->ctxqueue, ctx);
+			}
+
 			cur_queue = atomic_load_explicit(
 				&fnc->counters.ctxqueue_len,
 				memory_order_relaxed);