Merge remote-tracking 'frr/master' into SR-Routing

14 files changed, 412 insertions, 187 deletions

diff --git a/lib/frr_pthread.c b/lib/frr_pthread.c
index 19dfbaf54b..72b47ae5c3 100644
--- a/lib/frr_pthread.c
+++ b/lib/frr_pthread.c
@@ -1,5 +1,5 @@
 /*
- * Utilities and interfaces for managing POSIX threads
+ * Utilities and interfaces for managing POSIX threads within FRR.
  * Copyright (C) 2017  Cumulus Networks
  *
  * This program is free software; you can redistribute it and/or modify
@@ -25,78 +25,99 @@
 #include "memory.h"
 #include "hash.h"
 
-DEFINE_MTYPE_STATIC(LIB, FRR_PTHREAD, "FRR POSIX Thread");
+DEFINE_MTYPE(LIB, FRR_PTHREAD, "FRR POSIX Thread");
+DEFINE_MTYPE(LIB, PTHREAD_PRIM, "POSIX synchronization primitives");
 
+/* id for next created pthread */
 static unsigned int next_id = 0;
 
-/* Hash table of all frr_pthreads along with synchronization primitive(s) and
- * hash table callbacks.
- * ------------------------------------------------------------------------ */
-static struct hash *pthread_table;
-static pthread_mutex_t pthread_table_mtx = PTHREAD_MUTEX_INITIALIZER;
+/* default frr_pthread start/stop routine prototypes */
+static void *fpt_run(void *arg);
+static int fpt_halt(struct frr_pthread *fpt, void **res);
 
-/* pthread_table->hash_cmp */
-static int pthread_table_hash_cmp(const void *value1, const void *value2)
+/* default frr_pthread attributes */
+struct frr_pthread_attr frr_pthread_attr_default = {
+	.id = 0,
+	.start = fpt_run,
+	.stop = fpt_halt,
+	.name = "Anonymous",
+};
+
+/* hash table to keep track of all frr_pthreads */
+static struct hash *frr_pthread_hash;
+static pthread_mutex_t frr_pthread_hash_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+/* frr_pthread_hash->hash_cmp */
+static int frr_pthread_hash_cmp(const void *value1, const void *value2)
 {
 	const struct frr_pthread *tq1 = value1;
 	const struct frr_pthread *tq2 = value2;
 
-	return (tq1->id == tq2->id);
+	return (tq1->attr.id == tq2->attr.id);
 }
 
-/* pthread_table->hash_key */
-static unsigned int pthread_table_hash_key(void *value)
+/* frr_pthread_hash->hash_key */
+static unsigned int frr_pthread_hash_key(void *value)
 {
-	return ((struct frr_pthread *)value)->id;
+	return ((struct frr_pthread *)value)->attr.id;
 }
+
 /* ------------------------------------------------------------------------ */
 
 void frr_pthread_init()
 {
-	pthread_mutex_lock(&pthread_table_mtx);
+	pthread_mutex_lock(&frr_pthread_hash_mtx);
 	{
-		pthread_table = hash_create(pthread_table_hash_key,
-					    pthread_table_hash_cmp, NULL);
+		frr_pthread_hash = hash_create(frr_pthread_hash_key,
+					       frr_pthread_hash_cmp, NULL);
 	}
-	pthread_mutex_unlock(&pthread_table_mtx);
+	pthread_mutex_unlock(&frr_pthread_hash_mtx);
 }
 
 void frr_pthread_finish()
 {
-	pthread_mutex_lock(&pthread_table_mtx);
+	pthread_mutex_lock(&frr_pthread_hash_mtx);
 	{
-		hash_clean(pthread_table,
+		hash_clean(frr_pthread_hash,
 			   (void (*)(void *))frr_pthread_destroy);
-		hash_free(pthread_table);
+		hash_free(frr_pthread_hash);
 	}
-	pthread_mutex_unlock(&pthread_table_mtx);
+	pthread_mutex_unlock(&frr_pthread_hash_mtx);
 }
 
-struct frr_pthread *frr_pthread_new(const char *name, unsigned int id,
-				    void *(*start_routine)(void *),
-				    int (*stop_routine)(void **,
-							struct frr_pthread *))
+struct frr_pthread *frr_pthread_new(struct frr_pthread_attr *attr)
 {
 	static struct frr_pthread holder = {0};
 	struct frr_pthread *fpt = NULL;
 
-	pthread_mutex_lock(&pthread_table_mtx);
+	attr = attr ? attr : &frr_pthread_attr_default;
+
+	pthread_mutex_lock(&frr_pthread_hash_mtx);
 	{
-		holder.id = id;
-
-		if (!hash_lookup(pthread_table, &holder)) {
-			struct frr_pthread *fpt = XCALLOC(
-				MTYPE_FRR_PTHREAD, sizeof(struct frr_pthread));
-			fpt->id = id;
-			fpt->master = thread_master_create(name);
-			fpt->start_routine = start_routine;
-			fpt->stop_routine = stop_routine;
-			fpt->name = XSTRDUP(MTYPE_FRR_PTHREAD, name);
-
-			hash_get(pthread_table, fpt, hash_alloc_intern);
+		holder.attr.id = attr->id;
+
+		if (!hash_lookup(frr_pthread_hash, &holder)) {
+			fpt = XCALLOC(MTYPE_FRR_PTHREAD,
+				      sizeof(struct frr_pthread));
+			/* create new thread master */
+			fpt->master = thread_master_create(attr->name);
+			/* set attributes */
+			fpt->attr = *attr;
+			if (attr == &frr_pthread_attr_default)
+				fpt->attr.id = frr_pthread_get_id();
+			/* initialize startup synchronization primitives */
+			fpt->running_cond_mtx = XCALLOC(
+				MTYPE_PTHREAD_PRIM, sizeof(pthread_mutex_t));
+			fpt->running_cond = XCALLOC(MTYPE_PTHREAD_PRIM,
+						    sizeof(pthread_cond_t));
+			pthread_mutex_init(fpt->running_cond_mtx, NULL);
+			pthread_cond_init(fpt->running_cond, NULL);
+
+			/* insert into global thread hash */
+			hash_get(frr_pthread_hash, fpt, hash_alloc_intern);
 		}
 	}
-	pthread_mutex_unlock(&pthread_table_mtx);
+	pthread_mutex_unlock(&frr_pthread_hash_mtx);
 
 	return fpt;
 }
@@ -104,7 +125,11 @@ struct frr_pthread *frr_pthread_new(const char *name, unsigned int id,
 void frr_pthread_destroy(struct frr_pthread *fpt)
 {
 	thread_master_free(fpt->master);
-	XFREE(MTYPE_FRR_PTHREAD, fpt->name);
+
+	pthread_mutex_destroy(fpt->running_cond_mtx);
+	pthread_cond_destroy(fpt->running_cond);
+	XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond_mtx);
+	XFREE(MTYPE_PTHREAD_PRIM, fpt->running_cond);
 	XFREE(MTYPE_FRR_PTHREAD, fpt);
 }
 
@@ -113,74 +138,82 @@ struct frr_pthread *frr_pthread_get(unsigned int id)
 	static struct frr_pthread holder = {0};
 	struct frr_pthread *fpt;
 
-	pthread_mutex_lock(&pthread_table_mtx);
+	pthread_mutex_lock(&frr_pthread_hash_mtx);
 	{
-		holder.id = id;
-		fpt = hash_lookup(pthread_table, &holder);
+		holder.attr.id = id;
+		fpt = hash_lookup(frr_pthread_hash, &holder);
 	}
-	pthread_mutex_unlock(&pthread_table_mtx);
+	pthread_mutex_unlock(&frr_pthread_hash_mtx);
 
 	return fpt;
 }
 
-int frr_pthread_run(unsigned int id, const pthread_attr_t *attr, void *arg)
+int frr_pthread_run(struct frr_pthread *fpt, const pthread_attr_t *attr)
 {
-	struct frr_pthread *fpt = frr_pthread_get(id);
 	int ret;
 
-	if (!fpt)
-		return -1;
-
-	ret = pthread_create(&fpt->thread, attr, fpt->start_routine, arg);
+	ret = pthread_create(&fpt->thread, attr, fpt->attr.start, fpt);
 
-	/* Per pthread_create(3), the contents of fpt->thread are undefined if
-	 * pthread_create() did not succeed. Reset this value to zero. */
+	/*
+	 * Per pthread_create(3), the contents of fpt->thread are undefined if
+	 * pthread_create() did not succeed. Reset this value to zero.
+	 */
 	if (ret < 0)
 		memset(&fpt->thread, 0x00, sizeof(fpt->thread));
 
 	return ret;
 }
 
-/**
- * Calls the stop routine for the frr_pthread and resets any relevant fields.
- *
- * @param fpt - the frr_pthread to stop
- * @param result - pointer to result pointer
- * @return the return code from the stop routine
- */
-static int frr_pthread_stop_actual(struct frr_pthread *fpt, void **result)
+void frr_pthread_wait_running(struct frr_pthread *fpt)
 {
-	int ret = (*fpt->stop_routine)(result, fpt);
-	memset(&fpt->thread, 0x00, sizeof(fpt->thread));
-	return ret;
+	pthread_mutex_lock(fpt->running_cond_mtx);
+	{
+		while (!fpt->running)
+			pthread_cond_wait(fpt->running_cond,
+					  fpt->running_cond_mtx);
+	}
+	pthread_mutex_unlock(fpt->running_cond_mtx);
 }
 
-int frr_pthread_stop(unsigned int id, void **result)
+void frr_pthread_notify_running(struct frr_pthread *fpt)
 {
-	struct frr_pthread *fpt = frr_pthread_get(id);
-	return frr_pthread_stop_actual(fpt, result);
+	pthread_mutex_lock(fpt->running_cond_mtx);
+	{
+		fpt->running = true;
+		pthread_cond_signal(fpt->running_cond);
+	}
+	pthread_mutex_unlock(fpt->running_cond_mtx);
 }
 
-/**
+int frr_pthread_stop(struct frr_pthread *fpt, void **result)
+{
+	int ret = (*fpt->attr.stop)(fpt, result);
+	memset(&fpt->thread, 0x00, sizeof(fpt->thread));
+	return ret;
+}
+
+/*
  * Callback for hash_iterate to stop all frr_pthread's.
  */
 static void frr_pthread_stop_all_iter(struct hash_backet *hb, void *arg)
 {
 	struct frr_pthread *fpt = hb->data;
-	frr_pthread_stop_actual(fpt, NULL);
+	frr_pthread_stop(fpt, NULL);
 }
 
 void frr_pthread_stop_all()
 {
-	pthread_mutex_lock(&pthread_table_mtx);
+	pthread_mutex_lock(&frr_pthread_hash_mtx);
 	{
-		hash_iterate(pthread_table, frr_pthread_stop_all_iter, NULL);
+		hash_iterate(frr_pthread_hash, frr_pthread_stop_all_iter, NULL);
 	}
-	pthread_mutex_unlock(&pthread_table_mtx);
+	pthread_mutex_unlock(&frr_pthread_hash_mtx);
 }
 
 unsigned int frr_pthread_get_id()
 {
+	/* just a sanity check, this should never happen */
+	assert(next_id <= INT_MAX - 1);
 	return next_id++;
 }
 
@@ -188,3 +221,60 @@ void frr_pthread_yield(void)
 {
 	(void)sched_yield();
 }
+
+/*
+ * ----------------------------------------------------------------------------
+ * Default Event Loop
+ * ----------------------------------------------------------------------------
+ */
+
+/* dummy task for sleeper pipe */
+static int fpt_dummy(struct thread *thread)
+{
+	return 0;
+}
+
+/* poison pill task to end event loop */
+static int fpt_finish(struct thread *thread)
+{
+	struct frr_pthread *fpt = THREAD_ARG(thread);
+	atomic_store_explicit(&fpt->running, false, memory_order_relaxed);
+	return 0;
+}
+
+/* stop function, called from other threads to halt this one */
+static int fpt_halt(struct frr_pthread *fpt, void **res)
+{
+	thread_add_event(fpt->master, &fpt_finish, fpt, 0, NULL);
+	pthread_join(fpt->thread, res);
+	fpt = NULL;
+
+	return 0;
+}
+
+/* entry pthread function & main event loop */
+static void *fpt_run(void *arg)
+{
+	struct frr_pthread *fpt = arg;
+	fpt->master->owner = pthread_self();
+
+	int sleeper[2];
+	pipe(sleeper);
+	thread_add_read(fpt->master, &fpt_dummy, NULL, sleeper[0], NULL);
+
+	fpt->master->handle_signals = false;
+
+	frr_pthread_notify_running(fpt);
+
+	struct thread task;
+	while (atomic_load_explicit(&fpt->running, memory_order_relaxed)) {
+		if (thread_fetch(fpt->master, &task)) {
+			thread_call(&task);
+		}
+	}
+
+	close(sleeper[1]);
+	close(sleeper[0]);
+
+	return NULL;
+}
diff --git a/lib/frr_pthread.h b/lib/frr_pthread.h
index f6000340a7..2cc50196a8 100644
--- a/lib/frr_pthread.h
+++ b/lib/frr_pthread.h
@@ -1,5 +1,5 @@
 /*
- * Utilities and interfaces for managing POSIX threads
+ * Utilities and interfaces for managing POSIX threads within FRR.
  * Copyright (C) 2017  Cumulus Networks
  *
  * This program is free software; you can redistribute it and/or modify
@@ -21,36 +21,73 @@
 #define _FRR_PTHREAD_H
 
 #include <pthread.h>
+#include "frratomic.h"
+#include "memory.h"
 #include "thread.h"
 
+DECLARE_MTYPE(FRR_PTHREAD);
+DECLARE_MTYPE(PTHREAD_PRIM);
+
+struct frr_pthread;
+struct frr_pthread_attr;
+
+struct frr_pthread_attr {
+	int id;
+	void *(*start)(void *);
+	int (*stop)(struct frr_pthread *, void **);
+	const char *name;
+};
+
 struct frr_pthread {
 
 	/* pthread id */
 	pthread_t thread;
 
-	/* frr thread identifier */
-	unsigned int id;
-
 	/* thread master for this pthread's thread.c event loop */
 	struct thread_master *master;
 
-	/* start routine */
-	void *(*start_routine)(void *);
-
-	/* stop routine */
-	int (*stop_routine)(void **, struct frr_pthread *);
-
-	/* the (hopefully descriptive) name of this thread */
-	char *name;
+	/* caller-specified data; start & stop funcs, name, id */
+	struct frr_pthread_attr attr;
+
+	/*
+	 * Notification mechanism for allowing pthreads to notify their parents
+	 * when they are ready to do work. This mechanism has two associated
+	 * functions:
+	 *
+	 * - frr_pthread_wait_running()
+	 *   This function should be called by the spawning thread after
+	 *   frr_pthread_run(). It safely waits until the spawned thread
+	 *   indicates that is ready to do work by posting to the condition
+	 *   variable.
+	 *
+	 * - frr_pthread_notify_running()
+	 *   This function should be called by the spawned thread when it is
+	 *   ready to do work. It will wake up any threads waiting on the
+	 *   previously described condition.
+	 */
+	pthread_cond_t *running_cond;
+	pthread_mutex_t *running_cond_mtx;
+	_Atomic bool running;
+
+	/*
+	 * Fake thread-specific storage. No constraints on usage. Helpful when
+	 * creating reentrant pthread implementations. Can be used to pass
+	 * argument to pthread entry function.
+	 */
+	void *data;
 };
 
-/* Initializes this module.
+extern struct frr_pthread_attr frr_pthread_attr_default;
+
+/*
+ * Initializes this module.
  *
  * Must be called before using any of the other functions.
  */
 void frr_pthread_init(void);
 
-/* Uninitializes this module.
+/*
+ * Uninitializes this module.
  *
  * Destroys all registered frr_pthread's and internal data structures.
  *
@@ -59,34 +96,23 @@ void frr_pthread_init(void);
  */
 void frr_pthread_finish(void);
 
-/* Creates a new frr_pthread.
- *
- * If the provided ID is already assigned to an existing frr_pthread, the
- * return value will be NULL.
- *
- * @param name - the name of the thread. Doesn't have to be unique, but it
- * probably should be. This value is copied and may be safely free'd upon
- * return.
- *
- * @param id - the integral ID of the thread. MUST be unique. The caller may
- * use this id to retrieve the thread.
- *
- * @param start_routine - start routine for the pthread, will be passed to
- * pthread_create (see those docs for details)
+/*
+ * Creates a new frr_pthread with the given attributes.
  *
- * @param stop_routine - stop routine for the pthread, called to terminate the
- * thread. This function should gracefully stop the pthread and clean up any
- * thread-specific resources. The passed pointer is used to return a data
- * result.
+ * The 'attr' argument should be filled out with the desired attributes,
+ * including ID, start and stop functions and the desired name. Alternatively,
+ * if attr is NULL, the default attributes will be used. The pthread will be
+ * set up to run a basic threadmaster loop and the name will be "Anonymous".
+ * Scheduling tasks onto the threadmaster in the 'master' field of the returned
+ * frr_pthread will cause them to run on that pthread.
  *
+ * @param attr - the thread attributes
  * @return the created frr_pthread upon success, or NULL upon failure
  */
-struct frr_pthread *frr_pthread_new(const char *name, unsigned int id,
-				    void *(*start_routine)(void *),
-				    int (*stop_routine)(void **,
-							struct frr_pthread *));
+struct frr_pthread *frr_pthread_new(struct frr_pthread_attr *attr);
 
-/* Destroys an frr_pthread.
+/*
+ * Destroys an frr_pthread.
  *
  * Assumes that the associated pthread, if any, has already terminated.
  *
@@ -94,38 +120,66 @@ struct frr_pthread *frr_pthread_new(const char *name, unsigned int id,
  */
 void frr_pthread_destroy(struct frr_pthread *fpt);
 
-/* Gets an existing frr_pthread by its id.
+/*
+ * Gets an existing frr_pthread by its id.
  *
  * @return frr_thread associated with the provided id, or NULL on error
  */
 struct frr_pthread *frr_pthread_get(unsigned int id);
 
-/* Creates a new pthread and binds it to a frr_pthread.
+/*
+ * Creates a new pthread and binds it to a frr_pthread.
  *
  * This function is a wrapper for pthread_create. The first parameter is the
  * frr_pthread to bind the created pthread to. All subsequent arguments are
- * passed unmodified to pthread_create().
+ * passed unmodified to pthread_create(). The frr_pthread * provided will be
+ * used as the argument to the pthread entry function. If it is necessary to
+ * pass additional data, the 'data' field in the frr_pthread may be used.
  *
  * This function returns the same code as pthread_create(). If the value is
  * zero, the provided frr_pthread is bound to a running POSIX thread. If the
  * value is less than zero, the provided frr_pthread is guaranteed to be a
  * clean instance that may be susbsequently passed to frr_pthread_run().
  *
- * @param id - frr_pthread to bind the created pthread to
+ * @param fpt - frr_pthread * to run
  * @param attr - see pthread_create(3)
- * @param arg - see pthread_create(3)
  *
  * @return see pthread_create(3)
  */
-int frr_pthread_run(unsigned int id, const pthread_attr_t *attr, void *arg);
+int frr_pthread_run(struct frr_pthread *fpt, const pthread_attr_t *attr);
 
-/* Stops an frr_pthread with a result.
+/*
+ * Waits until the specified pthread has finished setting up and is ready to
+ * begin work.
  *
- * @param id - frr_pthread to stop
+ * If the pthread's code makes use of the startup synchronization mechanism,
+ * this function should be called before attempting to use the functionality
+ * exposed by the pthread. It waits until the 'running' condition is satisfied
+ * (see struct definition of frr_pthread).
+ *
+ * @param fpt - the frr_pthread * to wait on
+ */
+void frr_pthread_wait_running(struct frr_pthread *fpt);
+
+/*
+ * Notifies other pthreads that the calling thread has finished setting up and
+ * is ready to begin work.
+ *
+ * This will allow any other pthreads waiting in 'frr_pthread_wait_running' to
+ * proceed.
+ *
+ * @param fpt - the frr_pthread * that has finished setting up
+ */
+void frr_pthread_notify_running(struct frr_pthread *fpt);
+
+/*
+ * Stops a frr_pthread with a result.
+ *
+ * @param fpt - frr_pthread * to stop
  * @param result - where to store the thread's result, if any. May be NULL if a
  * result is not needed.
  */
-int frr_pthread_stop(unsigned int id, void **result);
+int frr_pthread_stop(struct frr_pthread *fpt, void **result);
 
 /* Stops all frr_pthread's. */
 void frr_pthread_stop_all(void);
@@ -133,7 +187,8 @@ void frr_pthread_stop_all(void);
 /* Yields the current thread of execution */
 void frr_pthread_yield(void);
 
-/* Returns a unique identifier for use with frr_pthread_new().
+/*
+ * Returns a unique identifier for use with frr_pthread_new().
  *
  * Internally, this is an integer that increments after each call to this
  * function. Because the number of pthreads created should never exceed INT_MAX
diff --git a/lib/if.c b/lib/if.c
index 8e6a9a6968..7866ddb8c4 100644
--- a/lib/if.c
+++ b/lib/if.c
@@ -210,6 +210,9 @@ void if_delete(struct interface *ifp)
 
 	if_link_params_free(ifp);
 
+	if (ifp->desc)
+		XFREE(MTYPE_TMP, ifp->desc);
+
 	XFREE(MTYPE_IF, ifp);
 }
 
@@ -219,6 +222,18 @@ struct interface *if_lookup_by_index(ifindex_t ifindex, vrf_id_t vrf_id)
 	struct vrf *vrf;
 	struct interface if_tmp;
 
+	if (vrf_id == VRF_UNKNOWN) {
+		struct interface *ifp;
+
+		RB_FOREACH(vrf, vrf_id_head, &vrfs_by_id) {
+			ifp = if_lookup_by_index(ifindex, vrf->vrf_id);
+			if (ifp)
+				return ifp;
+		}
+
+		return NULL;
+	}
+
 	vrf = vrf_lookup_by_id(vrf_id);
 	if (!vrf)
 		return NULL;
diff --git a/lib/if.h b/lib/if.h
index eb8af2041b..79f96a7c45 100644
--- a/lib/if.h
+++ b/lib/if.h
@@ -452,6 +452,13 @@ struct nbr_connected {
 /* Prototypes. */
 extern int if_cmp_name_func(char *, char *);
 
+/*
+ * Passing in VRF_UNKNOWN is a valid thing to do, unless we
+ * are creating a new interface.
+ *
+ * This is useful for vrf route-leaking.  So more than anything
+ * else think before you use VRF_UNKNOWN
+ */
 extern void if_update_to_new_vrf(struct interface *, vrf_id_t vrf_id);
 extern struct interface *if_create(const char *name,  vrf_id_t vrf_id);
 extern struct interface *if_lookup_by_index(ifindex_t, vrf_id_t vrf_id);
diff --git a/lib/mpls.h b/lib/mpls.h
index 9d103e669c..95882c26ec 100644
--- a/lib/mpls.h
+++ b/lib/mpls.h
@@ -89,6 +89,12 @@ typedef unsigned int mpls_lse_t;
 /* MPLS label value as a 32-bit (mostly we only care about the label value). */
 typedef unsigned int mpls_label_t;
 
+struct mpls_label_stack {
+	uint8_t num_labels;
+	uint8_t reserved[3];
+	mpls_label_t label[0]; /* 1 or more labels */
+};
+
 /* The MPLS explicit-null label is 0 which means when you memset a mpls_label_t
  * to zero you have set that variable to explicit-null which was probably not
  * your intent. The work-around is to use one bit to indicate if the
diff --git a/lib/nexthop.c b/lib/nexthop.c
index f6b2c9788d..f531f27302 100644
--- a/lib/nexthop.c
+++ b/lib/nexthop.c
@@ -124,7 +124,7 @@ const char *nexthop_type_to_str(enum nexthop_types_t nh_type)
  */
 int nexthop_labels_match(struct nexthop *nh1, struct nexthop *nh2)
 {
-	struct nexthop_label *nhl1, *nhl2;
+	struct mpls_label_stack *nhl1, *nhl2;
 
 	nhl1 = nh1->nh_label;
 	nhl2 = nh2->nh_label;
@@ -210,12 +210,12 @@ void nexthops_free(struct nexthop *nexthop)
 void nexthop_add_labels(struct nexthop *nexthop, enum lsp_types_t type,
 			u_int8_t num_labels, mpls_label_t *label)
 {
-	struct nexthop_label *nh_label;
+	struct mpls_label_stack *nh_label;
 	int i;
 
 	nexthop->nh_label_type = type;
 	nh_label = XCALLOC(MTYPE_NH_LABEL,
-			   sizeof(struct nexthop_label)
+			   sizeof(struct mpls_label_stack)
 				   + num_labels * sizeof(mpls_label_t));
 	nh_label->num_labels = num_labels;
 	for (i = 0; i < num_labels; i++)
diff --git a/lib/nexthop.h b/lib/nexthop.h
index a727f37057..753e66643d 100644
--- a/lib/nexthop.h
+++ b/lib/nexthop.h
@@ -55,13 +55,6 @@ enum blackhole_type {
 	((type) == NEXTHOP_TYPE_IFINDEX || (type) == NEXTHOP_TYPE_BLACKHOLE) \
 		? (type) : ((type) | 1)
 
-/* Nexthop label structure. */
-struct nexthop_label {
-	u_int8_t num_labels;
-	u_int8_t reserved[3];
-	mpls_label_t label[0]; /* 1 or more labels. */
-};
-
 /* Nexthop structure. */
 struct nexthop {
 	struct nexthop *next;
@@ -107,7 +100,7 @@ struct nexthop {
 	enum lsp_types_t nh_label_type;
 
 	/* Label(s) associated with this nexthop. */
-	struct nexthop_label *nh_label;
+	struct mpls_label_stack *nh_label;
 };
 
 /* The following for loop allows to iterate over the nexthop
diff --git a/lib/ns.h b/lib/ns.h
index c492d6600b..79b4cab04d 100644
--- a/lib/ns.h
+++ b/lib/ns.h
@@ -25,10 +25,11 @@
 #include "openbsd-tree.h"
 #include "linklist.h"
 
-typedef u_int16_t ns_id_t;
+typedef u_int32_t ns_id_t;
 
-/* The default NS ID */
+/* the default NS ID */
 #define NS_DEFAULT 0
+#define NS_UNKNOWN UINT32_MAX
 
 /* Default netns directory (Linux) */
 #define NS_RUN_DIR         "/var/run/netns"
diff --git a/lib/thread.c b/lib/thread.c
index d26db88550..9d64663d9c 100644
--- a/lib/thread.c
+++ b/lib/thread.c
@@ -919,6 +919,8 @@ struct thread *funcname_thread_add_event(struct thread_master *m,
  */
 static void thread_cancel_rw(struct thread_master *master, int fd, short state)
 {
+	bool found = false;
+
 	/* Cancel POLLHUP too just in case some bozo set it */
 	state |= POLLHUP;
 
@@ -926,8 +928,18 @@ static void thread_cancel_rw(struct thread_master *master, int fd, short state)
 	nfds_t i;
 
 	for (i = 0; i < master->handler.pfdcount; i++)
-		if (master->handler.pfds[i].fd == fd)
+		if (master->handler.pfds[i].fd == fd) {
+			found = true;
 			break;
+		}
+
+	if (!found) {
+		zlog_debug(
+			"[!] Received cancellation request for nonexistent rw job");
+		zlog_debug("[!] threadmaster: %s | fd: %d",
+			 master->name ? master->name : "", fd);
+		return;
+	}
 
 	/* NOT out event. */
 	master->handler.pfds[i].events &= ~(state);
diff --git a/lib/vrf.c b/lib/vrf.c
index 3c34b95262..2fa3a9c0ef 100644
--- a/lib/vrf.c
+++ b/lib/vrf.c
@@ -94,7 +94,7 @@ struct vrf *vrf_get(vrf_id_t vrf_id, const char *name)
 	int new = 0;
 
 	if (debug_vrf)
-		zlog_debug("VRF_GET: %s(%d)", name, vrf_id);
+		zlog_debug("VRF_GET: %s(%u)", name, vrf_id);
 
 	/* Nothing to see, move along here */
 	if (!name && vrf_id == VRF_UNKNOWN)
@@ -267,8 +267,8 @@ void *vrf_info_lookup(vrf_id_t vrf_id)
  * VRF bit-map
  */
 
-#define VRF_BITMAP_NUM_OF_GROUPS            8
-#define VRF_BITMAP_NUM_OF_BITS_IN_GROUP (UINT16_MAX / VRF_BITMAP_NUM_OF_GROUPS)
+#define VRF_BITMAP_NUM_OF_GROUPS            1024
+#define VRF_BITMAP_NUM_OF_BITS_IN_GROUP (UINT32_MAX / VRF_BITMAP_NUM_OF_GROUPS)
 #define VRF_BITMAP_NUM_OF_BYTES_IN_GROUP                                       \
 	(VRF_BITMAP_NUM_OF_BITS_IN_GROUP / CHAR_BIT + 1) /* +1 for ensure */
 
@@ -355,7 +355,7 @@ static void vrf_autocomplete(vector comps, struct cmd_token *token)
 	struct vrf *vrf = NULL;
 
 	RB_FOREACH (vrf, vrf_name_head, &vrfs_by_name) {
-		if (vrf->vrf_id != 0)
+		if (vrf->vrf_id != VRF_DEFAULT)
 			vector_set(comps, XSTRDUP(MTYPE_COMPLETION, vrf->name));
 	}
 }
diff --git a/lib/vrf.h b/lib/vrf.h
index 9afca4c6fb..7e625769e7 100644
--- a/lib/vrf.h
+++ b/lib/vrf.h
@@ -32,8 +32,7 @@
 
 /* The default VRF ID */
 #define VRF_DEFAULT 0
-#define VRF_UNKNOWN UINT16_MAX
-#define VRF_ALL UINT16_MAX - 1
+#define VRF_UNKNOWN UINT32_MAX
 
 /* Pending: May need to refine this. */
 #ifndef IFLA_VRF_MAX
diff --git a/lib/zclient.c b/lib/zclient.c
index 4177ce1a71..d3717d0cdd 100644
--- a/lib/zclient.c
+++ b/lib/zclient.c
@@ -291,7 +291,7 @@ void zclient_create_header(struct stream *s, uint16_t command, vrf_id_t vrf_id)
 	stream_putw(s, ZEBRA_HEADER_SIZE);
 	stream_putc(s, ZEBRA_HEADER_MARKER);
 	stream_putc(s, ZSERV_VERSION);
-	stream_putw(s, vrf_id);
+	stream_putl(s, vrf_id);
 	stream_putw(s, command);
 }
 
@@ -306,7 +306,7 @@ int zclient_read_header(struct stream *s, int sock, u_int16_t *size,
 	*size -= ZEBRA_HEADER_SIZE;
 	STREAM_GETC(s, *marker);
 	STREAM_GETC(s, *version);
-	STREAM_GETW(s, *vrf_id);
+	STREAM_GETL(s, *vrf_id);
 	STREAM_GETW(s, *cmd);
 
 	if (*version != ZSERV_VERSION || *marker != ZEBRA_HEADER_MARKER) {
@@ -389,25 +389,28 @@ void zclient_send_reg_requests(struct zclient *zclient, vrf_id_t vrf_id)
 			       vrf_id);
 
 	/* Flush all redistribute request. */
-	if (vrf_id == VRF_DEFAULT)
-		for (afi = AFI_IP; afi < AFI_MAX; afi++)
-			for (i = 0; i < ZEBRA_ROUTE_MAX; i++)
-				if (zclient->mi_redist[afi][i].enabled) {
-					struct listnode *node;
-					u_short *id;
-
-					for (ALL_LIST_ELEMENTS_RO(
-						     zclient->mi_redist[afi][i]
-							     .instances,
-						     node, id))
-						if (!(i == zclient->redist_default
-						      && *id == zclient->instance))
-							zebra_redistribute_send(
-								ZEBRA_REDISTRIBUTE_ADD,
-								zclient, afi, i,
-								*id,
-								VRF_DEFAULT);
-				}
+	if (vrf_id == VRF_DEFAULT) {
+		for (afi = AFI_IP; afi < AFI_MAX; afi++) {
+			for (i = 0; i < ZEBRA_ROUTE_MAX; i++) {
+				if (!zclient->mi_redist[afi][i].enabled)
+					continue;
+
+				struct listnode *node;
+				u_short *id;
+
+				for (ALL_LIST_ELEMENTS_RO(
+					     zclient->mi_redist[afi][i]
+					     .instances, node, id))
+					if (!(i == zclient->redist_default
+					      && *id == zclient->instance))
+						zebra_redistribute_send(
+							ZEBRA_REDISTRIBUTE_ADD,
+							zclient, afi, i,
+							*id,
+							VRF_DEFAULT);
+			}
+		}
+	}
 
 	/* Flush all redistribute request. */
 	for (afi = AFI_IP; afi < AFI_MAX; afi++)
@@ -447,29 +450,32 @@ void zclient_send_dereg_requests(struct zclient *zclient, vrf_id_t vrf_id)
 
 	/* Set unwanted redistribute route. */
 	for (afi = AFI_IP; afi < AFI_MAX; afi++)
-		vrf_bitmap_set(zclient->redist[afi][zclient->redist_default],
-			       vrf_id);
+		vrf_bitmap_unset(zclient->redist[afi][zclient->redist_default],
+				 vrf_id);
 
 	/* Flush all redistribute request. */
-	if (vrf_id == VRF_DEFAULT)
-		for (afi = AFI_IP; afi < AFI_MAX; afi++)
-			for (i = 0; i < ZEBRA_ROUTE_MAX; i++)
-				if (zclient->mi_redist[afi][i].enabled) {
-					struct listnode *node;
-					u_short *id;
-
-					for (ALL_LIST_ELEMENTS_RO(
-						     zclient->mi_redist[afi][i]
-							     .instances,
-						     node, id))
-						if (!(i == zclient->redist_default
-						      && *id == zclient->instance))
-							zebra_redistribute_send(
-								ZEBRA_REDISTRIBUTE_DELETE,
-								zclient, afi, i,
-								*id,
-								VRF_DEFAULT);
-				}
+	if (vrf_id == VRF_DEFAULT) {
+		for (afi = AFI_IP; afi < AFI_MAX; afi++) {
+			for (i = 0; i < ZEBRA_ROUTE_MAX; i++) {
+				if (!zclient->mi_redist[afi][i].enabled)
+					continue;
+
+				struct listnode *node;
+				u_short *id;
+
+				for (ALL_LIST_ELEMENTS_RO(
+					     zclient->mi_redist[afi][i]
+					     .instances, node, id))
+					if (!(i == zclient->redist_default
+					      && *id == zclient->instance))
+						zebra_redistribute_send(
+							ZEBRA_REDISTRIBUTE_DELETE,
+							zclient, afi, i,
+							*id,
+							VRF_DEFAULT);
+			}
+		}
+	}
 
 	/* Flush all redistribute request. */
 	for (afi = AFI_IP; afi < AFI_MAX; afi++)
@@ -608,6 +614,33 @@ static int zclient_connect(struct thread *t)
 	return zclient_start(zclient);
 }
 
+int zclient_send_rnh(struct zclient *zclient, int command, struct prefix *p,
+		     bool exact_match, vrf_id_t vrf_id)
+{
+	struct stream *s;
+
+	s = zclient->obuf;
+	stream_reset(s);
+	zclient_create_header(s, command, vrf_id);
+	stream_putc(s, (exact_match) ? 1 : 0);
+
+	stream_putw(s, PREFIX_FAMILY(p));
+	stream_putc(s, p->prefixlen);
+	switch (PREFIX_FAMILY(p)) {
+	case AF_INET:
+		stream_put_in_addr(s, &p->u.prefix4);
+		break;
+	case AF_INET6:
+		stream_put(s, &(p->u.prefix6), 16);
+		break;
+	default:
+		break;
+	}
+	stream_putw_at(s, 0, stream_get_endp(s));
+
+	return zclient_send_message(zclient);
+}
+
 /*
  * "xdr_encode"-like interface that allows daemon (client) to send
  * a message to zebra server for a route that needs to be
@@ -942,6 +975,8 @@ int zapi_route_encode(u_char cmd, struct stream *s, struct zapi_route *api)
 		}
 
 		stream_putw(s, api->nexthop_num);
+		if (api->nexthop_num)
+			stream_putw(s, api->nh_vrf_id);
 
 		for (i = 0; i < api->nexthop_num; i++) {
 			api_nh = &api->nexthops[i];
@@ -1091,6 +1126,9 @@ int zapi_route_decode(struct stream *s, struct zapi_route *api)
 			return -1;
 		}
 
+		if (api->nexthop_num)
+			STREAM_GETW(s, api->nh_vrf_id);
+
 		for (i = 0; i < api->nexthop_num; i++) {
 			api_nh = &api->nexthops[i];
 
@@ -1677,7 +1715,7 @@ struct interface *zebra_interface_vrf_update_read(struct stream *s,
 {
 	unsigned int ifindex;
 	struct interface *ifp;
-	vrf_id_t new_id = VRF_DEFAULT;
+	vrf_id_t new_id;
 
 	/* Get interface index. */
 	ifindex = stream_getl(s);
@@ -2043,7 +2081,7 @@ static int zclient_read(struct thread *thread)
 	length = stream_getw(zclient->ibuf);
 	marker = stream_getc(zclient->ibuf);
 	version = stream_getc(zclient->ibuf);
-	vrf_id = stream_getw(zclient->ibuf);
+	vrf_id = stream_getl(zclient->ibuf);
 	command = stream_getw(zclient->ibuf);
 
 	if (marker != ZEBRA_HEADER_MARKER || version != ZSERV_VERSION) {
@@ -2346,9 +2384,9 @@ void zclient_interface_set_master(struct zclient *client,
 
 	zclient_create_header(s, ZEBRA_INTERFACE_SET_MASTER, master->vrf_id);
 
-	stream_putw(s, master->vrf_id);
+	stream_putl(s, master->vrf_id);
 	stream_putl(s, master->ifindex);
-	stream_putw(s, slave->vrf_id);
+	stream_putl(s, slave->vrf_id);
 	stream_putl(s, slave->ifindex);
 
 	stream_putw_at(s, 0, stream_get_endp(s));
diff --git a/lib/zclient.h b/lib/zclient.h
index cc34fd9d2c..df9e5f438d 100644
--- a/lib/zclient.h
+++ b/lib/zclient.h
@@ -40,7 +40,7 @@
 #define ZEBRA_MAX_PACKET_SIZ          4096
 
 /* Zebra header size. */
-#define ZEBRA_HEADER_SIZE             8
+#define ZEBRA_HEADER_SIZE             10
 
 /* special socket path name to use TCP
  * @ is used as first character because that's abstract socket names on Linux
@@ -227,7 +227,7 @@ struct zserv_header {
 			 * always set to 255 in new zserv.
 			 */
 	uint8_t version;
-#define ZSERV_VERSION	4
+#define ZSERV_VERSION	5
 	vrf_id_t vrf_id;
 	uint16_t command;
 };
@@ -281,6 +281,7 @@ struct zapi_route {
 	u_int32_t mtu;
 
 	vrf_id_t vrf_id;
+	vrf_id_t nh_vrf_id;
 
 	struct ethaddr rmac;
 };
@@ -420,6 +421,11 @@ extern struct interface *zebra_interface_vrf_update_read(struct stream *s,
 							 vrf_id_t *new_vrf_id);
 extern void zebra_interface_if_set_value(struct stream *, struct interface *);
 extern void zebra_router_id_update_read(struct stream *s, struct prefix *rid);
+
+#if CONFDATE > 20180823
+CPP_NOTICE("zapi_ipv4_route, zapi_ipv6_route, zapi_ipv4_route_ipv6_nexthop as well as the zapi_ipv4 and zapi_ipv6 data structures should be removed now");
+#endif
+
 extern int zapi_ipv4_route(u_char, struct zclient *, struct prefix_ipv4 *,
 			   struct zapi_ipv4 *) __attribute__((deprecated));
 
@@ -478,6 +484,9 @@ extern int zapi_ipv4_route_ipv6_nexthop(u_char, struct zclient *,
 					struct zapi_ipv6 *)
 	__attribute__((deprecated));
 extern int zclient_route_send(u_char, struct zclient *, struct zapi_route *);
+extern int zclient_send_rnh(struct zclient *zclient, int command,
+			    struct prefix *p, bool exact_match,
+			    vrf_id_t vrf_id);
 extern int zapi_route_encode(u_char, struct stream *, struct zapi_route *);
 extern int zapi_route_decode(struct stream *, struct zapi_route *);
 bool zapi_route_notify_decode(struct stream *s, struct prefix *p,
diff --git a/lib/zebra.h b/lib/zebra.h
index 1eb0c56252..b9a795d160 100644
--- a/lib/zebra.h
+++ b/lib/zebra.h
@@ -486,7 +486,7 @@ typedef u_int16_t zebra_size_t;
 typedef u_int16_t zebra_command_t;
 
 /* VRF ID type. */
-typedef u_int16_t vrf_id_t;
+typedef uint32_t vrf_id_t;
 
 typedef uint32_t route_tag_t;
 #define ROUTE_TAG_MAX UINT32_MAX