lib: serialize pthread startup

Add a new condition var and mutex to serialize pthread startup.
When a new pthread is started, it will wait very early on for the
parent pthread to permit it to run. This ensures that that the
ordering between parent and child is predictable.

Signed-off-by: Mark Stapp <mjs@cisco.com>

diff --git a/lib/frr_pthread.c b/lib/frr_pthread.c
index 1ffa5934aa5bd0ef2b32794104255127a86943fe..3a4bc712fc529a35c81e62d104e723f06d3702f4 100644 (file)
--- a/lib/frr_pthread.c
+++ b/lib/frr_pthread.c
@@ -92,9 +92,14 @@ struct frr_pthread *frr_pthread_new(const struct frr_pthread_attr *attr,
                MTYPE_PTHREAD_PRIM, sizeof(pthread_mutex_t));
        fpt->running_cond = XCALLOC(MTYPE_PTHREAD_PRIM,
                                    sizeof(pthread_cond_t));
+
        pthread_mutex_init(fpt->running_cond_mtx, NULL);
        pthread_cond_init(fpt->running_cond, NULL);
 
+       pthread_mutex_init(&fpt->startup_cond_mtx, NULL);
+       pthread_cond_init(&fpt->startup_cond, NULL);
+       fpt->started = false;
+
        frr_with_mutex (&frr_pthread_list_mtx) {
                listnode_add(frr_pthread_list, fpt);
        }
@@ -108,6 +113,8 @@ static void frr_pthread_destroy_nolock(struct frr_pthread *fpt)
        pthread_mutex_destroy(&fpt->mtx);
        pthread_mutex_destroy(fpt->running_cond_mtx);
        pthread_cond_destroy(fpt->running_cond);
+       pthread_mutex_destroy(&fpt->startup_cond_mtx);
+       pthread_cond_destroy(&fpt->startup_cond);
        XFREE(MTYPE_FRR_PTHREAD, fpt->name);
        XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond_mtx);
        XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond);
@@ -140,11 +147,34 @@ int frr_pthread_set_name(struct frr_pthread *fpt)
        return ret;
 }
 
+/* New pthread waits before running */
+static void frr_pthread_wait_startup(struct frr_pthread *fpt)
+{
+       frr_with_mutex (&fpt->startup_cond_mtx) {
+               while (!fpt->started)
+                       pthread_cond_wait(&fpt->startup_cond,
+                                         &fpt->startup_cond_mtx);
+       }
+}
+
+/* Parent pthread allows new pthread to start running */
+static void frr_pthread_notify_startup(struct frr_pthread *fpt)
+{
+       frr_with_mutex (&fpt->startup_cond_mtx) {
+               fpt->started = true;
+               pthread_cond_signal(&fpt->startup_cond);
+       }
+}
+
 static void *frr_pthread_inner(void *arg)
 {
        struct frr_pthread *fpt = arg;
 
+       /* The new pthead waits until the parent allows it to continue. */
+       frr_pthread_wait_startup(fpt);
+
        rcu_thread_start(fpt->rcu_thread);
+
        return fpt->attr.start(fpt);
 }
 
@@ -169,6 +199,9 @@ int frr_pthread_run(struct frr_pthread *fpt, const pthread_attr_t *attr)
        /* Restore caller's signals */
        pthread_sigmask(SIG_SETMASK, &oldsigs, NULL);
 
+       /* Allow new child pthread to start */
+       frr_pthread_notify_startup(fpt);
+
        /*
         * Per pthread_create(3), the contents of fpt->thread are undefined if
         * pthread_create() did not succeed. Reset this value to zero.
@@ -250,6 +283,8 @@ int frr_pthread_non_controlled_startup(pthread_t thread, const char *name,
 
        fpt->thread = thread;
        fpt->rcu_thread = rcu_thread;
+       fpt->started = true;
+
        frr_pthread_inner(fpt);
 
        return 0;

diff --git a/lib/frr_pthread.h b/lib/frr_pthread.h
index 1e1b8d7fd3e21dd4618bc2ddd26105d67235be9e..bb751b707186ecfc4d60718e7478c12154be6906 100644 (file)
--- a/lib/frr_pthread.h
+++ b/lib/frr_pthread.h
@@ -46,6 +46,17 @@ struct frr_pthread {
        /* caller-specified data; start & stop funcs, name, id */
        struct frr_pthread_attr attr;
 
+       /*
+        * Startup serialization: newly-started pthreads wait at a point
+        * very early in life so that there isn't a race with the
+        * starting pthread. The OS 'start' apis don't make any guarantees
+        * about which pthread runs first - the existing pthread that has
+        * called the 'start' api, or the new pthread that is just starting.
+        */
+       pthread_cond_t startup_cond;
+       pthread_mutex_t startup_cond_mtx;
+       atomic_bool started;
+
        /*
         * Notification mechanism for allowing pthreads to notify their parents
         * when they are ready to do work. This mechanism has two associated