lib: serialize pthread startup

Add a new condition var and mutex to serialize pthread startup.
When a new pthread is started, it will wait very early on for the
parent pthread to permit it to run. This ensures that that the
ordering between parent and child is predictable.

Signed-off-by: Mark Stapp <mjs@cisco.com>

1 files changed, 35 insertions, 0 deletions

diff --git a/lib/frr_pthread.c b/lib/frr_pthread.c
index 1ffa5934aa..3a4bc712fc 100644
--- a/lib/frr_pthread.c
+++ b/lib/frr_pthread.c
@@ -92,9 +92,14 @@ struct frr_pthread *frr_pthread_new(const struct frr_pthread_attr *attr,
 		MTYPE_PTHREAD_PRIM, sizeof(pthread_mutex_t));
 	fpt->running_cond = XCALLOC(MTYPE_PTHREAD_PRIM,
 				    sizeof(pthread_cond_t));
+
 	pthread_mutex_init(fpt->running_cond_mtx, NULL);
 	pthread_cond_init(fpt->running_cond, NULL);
 
+	pthread_mutex_init(&fpt->startup_cond_mtx, NULL);
+	pthread_cond_init(&fpt->startup_cond, NULL);
+	fpt->started = false;
+
 	frr_with_mutex (&frr_pthread_list_mtx) {
 		listnode_add(frr_pthread_list, fpt);
 	}
@@ -108,6 +113,8 @@ static void frr_pthread_destroy_nolock(struct frr_pthread *fpt)
 	pthread_mutex_destroy(&fpt->mtx);
 	pthread_mutex_destroy(fpt->running_cond_mtx);
 	pthread_cond_destroy(fpt->running_cond);
+	pthread_mutex_destroy(&fpt->startup_cond_mtx);
+	pthread_cond_destroy(&fpt->startup_cond);
 	XFREE(MTYPE_FRR_PTHREAD, fpt->name);
 	XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond_mtx);
 	XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond);
@@ -140,11 +147,34 @@ int frr_pthread_set_name(struct frr_pthread *fpt)
 	return ret;
 }
 
+/* New pthread waits before running */
+static void frr_pthread_wait_startup(struct frr_pthread *fpt)
+{
+	frr_with_mutex (&fpt->startup_cond_mtx) {
+		while (!fpt->started)
+			pthread_cond_wait(&fpt->startup_cond,
+					  &fpt->startup_cond_mtx);
+	}
+}
+
+/* Parent pthread allows new pthread to start running */
+static void frr_pthread_notify_startup(struct frr_pthread *fpt)
+{
+	frr_with_mutex (&fpt->startup_cond_mtx) {
+		fpt->started = true;
+		pthread_cond_signal(&fpt->startup_cond);
+	}
+}
+
 static void *frr_pthread_inner(void *arg)
 {
 	struct frr_pthread *fpt = arg;
 
+	/* The new pthead waits until the parent allows it to continue. */
+	frr_pthread_wait_startup(fpt);
+
 	rcu_thread_start(fpt->rcu_thread);
+
 	return fpt->attr.start(fpt);
 }
 
@@ -169,6 +199,9 @@ int frr_pthread_run(struct frr_pthread *fpt, const pthread_attr_t *attr)
 	/* Restore caller's signals */
 	pthread_sigmask(SIG_SETMASK, &oldsigs, NULL);
 
+	/* Allow new child pthread to start */
+	frr_pthread_notify_startup(fpt);
+
 	/*
 	 * Per pthread_create(3), the contents of fpt->thread are undefined if
 	 * pthread_create() did not succeed. Reset this value to zero.
@@ -250,6 +283,8 @@ int frr_pthread_non_controlled_startup(pthread_t thread, const char *name,
 
 	fpt->thread = thread;
 	fpt->rcu_thread = rcu_thread;
+	fpt->started = true;
+
 	frr_pthread_inner(fpt);
 
 	return 0;