From d166308be009537caf733dbe0b55fae7553726a1 Mon Sep 17 00:00:00 2001 From: Mark Stapp Date: Wed, 14 Jul 2021 13:06:41 -0400 Subject: [PATCH] zebra: use the dataplane to read netlink intf addr changes Read incoming interface address change notifications in the dplane pthread; enqueue the events to the main pthread for processing. This is netlink-only for now - the bsd kernel socket path remains unchanged. Signed-off-by: Mark Stapp --- zebra/interface.c | 10 ++++ zebra/interface.h | 1 + zebra/kernel_netlink.c | 77 +++++++++++++++++++------ zebra/kernel_socket.c | 8 +++ zebra/rt.h | 5 ++ zebra/zebra_dplane.c | 126 ++++++++++++++++++++++++++++++++++++++++- zebra/zebra_ns.c | 3 + zebra/zebra_rib.c | 7 ++- 8 files changed, 217 insertions(+), 20 deletions(-) diff --git a/zebra/interface.c b/zebra/interface.c index 328ef3fa41..ec1f16f671 100644 --- a/zebra/interface.c +++ b/zebra/interface.c @@ -1205,6 +1205,16 @@ void zebra_if_set_protodown(struct interface *ifp, bool down) #endif } +/* Handler for incoming intf address change events */ +void zebra_if_addr_update_ctx(struct zebra_dplane_ctx *ctx) +{ +#ifdef HAVE_NETLINK + netlink_interface_addr_ctx(ctx); +#else + dplane_ctx_fini(&ctx); +#endif /* HAVE_NETLINK */ +} + /* Dump if address information to vty. */ static void connected_dump_vty(struct vty *vty, json_object *json, struct connected *connected) diff --git a/zebra/interface.h b/zebra/interface.h index d86bc68ef0..23e22bdda8 100644 --- a/zebra/interface.h +++ b/zebra/interface.h @@ -513,6 +513,7 @@ extern void zebra_l2_map_slave_to_bond(struct zebra_if *zif, vrf_id_t vrf); extern void zebra_l2_unmap_slave_from_bond(struct zebra_if *zif); extern const char *zebra_protodown_rc_str(enum protodown_reasons protodown_rc, char *pd_buf, uint32_t pd_buf_len); +void zebra_if_addr_update_ctx(struct zebra_dplane_ctx *ctx); #ifdef HAVE_PROC_NET_DEV extern void ifstat_update_proc(void); diff --git a/zebra/kernel_netlink.c b/zebra/kernel_netlink.c index f50fd74ac1..602bdc1dc5 100644 --- a/zebra/kernel_netlink.c +++ b/zebra/kernel_netlink.c @@ -324,6 +324,10 @@ static int netlink_socket(struct nlsock *nl, unsigned long groups, return ret; } +/* + * Dispatch an incoming netlink message; used by the zebra main pthread's + * netlink event reader. + */ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id, int startup) { @@ -345,10 +349,6 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id, return netlink_link_change(h, ns_id, startup); case RTM_DELLINK: return netlink_link_change(h, ns_id, startup); - case RTM_NEWADDR: - return netlink_interface_addr(h, ns_id, startup); - case RTM_DELADDR: - return netlink_interface_addr(h, ns_id, startup); case RTM_NEWNEIGH: case RTM_DELNEIGH: case RTM_GETNEIGH: @@ -361,6 +361,12 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id, return netlink_nexthop_change(h, ns_id, startup); case RTM_DELNEXTHOP: return netlink_nexthop_change(h, ns_id, startup); + + /* Messages handled in the dplane thread */ + case RTM_NEWADDR: + case RTM_DELADDR: + return 0; + default: /* * If we have received this message then @@ -378,6 +384,32 @@ static int netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id, return 0; } +/* + * Dispatch an incoming netlink message; used by the dataplane pthread's + * netlink event reader code. + */ +static int dplane_netlink_information_fetch(struct nlmsghdr *h, ns_id_t ns_id, + int startup) +{ + /* + * Dispatch the incoming messages that the dplane pthread handles + */ + switch (h->nlmsg_type) { + case RTM_NEWADDR: + case RTM_DELADDR: + return netlink_interface_addr_dplane(h, ns_id, startup); + + /* TODO */ + case RTM_NEWLINK: + case RTM_DELLINK: + + default: + break; + } + + return 0; +} + static int kernel_read(struct thread *thread) { struct zebra_ns *zns = (struct zebra_ns *)THREAD_ARG(thread); @@ -388,13 +420,24 @@ static int kernel_read(struct thread *thread) netlink_parse_info(netlink_information_fetch, &zns->netlink, &dp_info, 5, 0); - zns->t_netlink = NULL; + thread_add_read(zrouter.master, kernel_read, zns, zns->netlink.sock, &zns->t_netlink); return 0; } +/* + * Called by the dplane pthread to read incoming OS messages and dispatch them. + */ +int kernel_dplane_read(struct zebra_dplane_info *info) +{ + netlink_parse_info(dplane_netlink_information_fetch, &info->nls, info, + 5, 0); + + return 0; +} + /* * Filter out messages from self that occur on listener socket, * caused by our actions on the command socket(s) @@ -476,9 +519,8 @@ static void netlink_install_filter(int sock, uint32_t pid, uint32_t dplane_pid) safe_strerror(errno)); } -void netlink_parse_rtattr_flags(struct rtattr **tb, int max, - struct rtattr *rta, int len, - unsigned short flags) +void netlink_parse_rtattr_flags(struct rtattr **tb, int max, struct rtattr *rta, + int len, unsigned short flags) { unsigned short type; @@ -800,8 +842,7 @@ static int netlink_recv_msg(const struct nlsock *nl, struct msghdr msg, * ignored, -1 otherwise. */ static int netlink_parse_error(const struct nlsock *nl, struct nlmsghdr *h, - const struct zebra_dplane_info *zns, - bool startup) + bool is_cmd, bool startup) { struct nlmsgerr *err = (struct nlmsgerr *)NLMSG_DATA(h); int errnum = err->error; @@ -834,7 +875,7 @@ static int netlink_parse_error(const struct nlsock *nl, struct nlmsghdr *h, } /* Deal with errors that occur because of races in link handling. */ - if (zns->is_cmd + if (is_cmd && ((msg_type == RTM_DELROUTE && (-errnum == ENODEV || -errnum == ESRCH)) || (msg_type == RTM_NEWROUTE @@ -853,7 +894,7 @@ static int netlink_parse_error(const struct nlsock *nl, struct nlmsghdr *h, * do not log these as an error. */ if (msg_type == RTM_DELNEIGH - || (zns->is_cmd && msg_type == RTM_NEWROUTE + || (is_cmd && msg_type == RTM_NEWROUTE && (-errnum == ESRCH || -errnum == ENETUNREACH))) { /* * This is known to happen in some situations, don't log as @@ -925,8 +966,9 @@ int netlink_parse_info(int (*filter)(struct nlmsghdr *, ns_id_t, int), /* Error handling. */ if (h->nlmsg_type == NLMSG_ERROR) { - int err = netlink_parse_error(nl, h, zns, - startup); + int err = netlink_parse_error( + nl, h, zns->is_cmd, startup); + if (err == 1) { if (!(h->nlmsg_flags & NLM_F_MULTI)) return 0; @@ -938,8 +980,8 @@ int netlink_parse_info(int (*filter)(struct nlmsghdr *, ns_id_t, int), /* OK we got netlink message. */ if (IS_ZEBRA_DEBUG_KERNEL) zlog_debug( - "netlink_parse_info: %s type %s(%u), len=%d, seq=%u, pid=%u", - nl->name, + "%s: %s type %s(%u), len=%d, seq=%u, pid=%u", + __func__, nl->name, nl_msg_type_to_str(h->nlmsg_type), h->nlmsg_type, h->nlmsg_len, h->nlmsg_seq, h->nlmsg_pid); @@ -1141,7 +1183,8 @@ static int nl_batch_read_resp(struct nl_batch *bth) } if (h->nlmsg_type == NLMSG_ERROR) { - int err = netlink_parse_error(nl, h, bth->zns, 0); + int err = netlink_parse_error(nl, h, bth->zns->is_cmd, + false); if (err == -1) dplane_ctx_set_status( diff --git a/zebra/kernel_socket.c b/zebra/kernel_socket.c index 2af6ebe6b1..d9c69ceb6d 100644 --- a/zebra/kernel_socket.c +++ b/zebra/kernel_socket.c @@ -1457,6 +1457,14 @@ void kernel_terminate(struct zebra_ns *zns, bool complete) return; } +/* + * Called by the dplane pthread to read incoming OS messages and dispatch them. + */ +int kernel_dplane_read(struct zebra_dplane_info *info) +{ + return 0; +} + void kernel_update_multi(struct dplane_ctx_q *ctx_list) { struct zebra_dplane_ctx *ctx; diff --git a/zebra/rt.h b/zebra/rt.h index 929a44ade7..90148d2c0d 100644 --- a/zebra/rt.h +++ b/zebra/rt.h @@ -110,6 +110,11 @@ extern int kernel_del_mac_nhg(uint32_t nhg_id); */ extern void kernel_update_multi(struct dplane_ctx_q *ctx_list); +/* + * Called by the dplane pthread to read incoming OS messages and dispatch them. + */ +int kernel_dplane_read(struct zebra_dplane_info *info); + #ifdef __cplusplus } #endif diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index ed374fa73c..ab06ea6438 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -37,11 +37,12 @@ #include "zebra/zebra_pbr.h" #include "printfrr.h" -/* Memory type for context blocks */ +/* Memory types */ DEFINE_MTYPE_STATIC(ZEBRA, DP_CTX, "Zebra DPlane Ctx"); DEFINE_MTYPE_STATIC(ZEBRA, DP_INTF, "Zebra DPlane Intf"); DEFINE_MTYPE_STATIC(ZEBRA, DP_PROV, "Zebra DPlane Provider"); DEFINE_MTYPE_STATIC(ZEBRA, DP_NETFILTER, "Zebra Netfilter Internal Object"); +DEFINE_MTYPE_STATIC(ZEBRA, DP_NS, "DPlane NSes"); #ifndef AOK # define AOK 0 @@ -402,6 +403,19 @@ struct zebra_dplane_provider { TAILQ_ENTRY(zebra_dplane_provider) dp_prov_link; }; +/* Declare types for list of zns info objects */ +PREDECL_DLIST(zns_info_list); + +struct dplane_zns_info { + struct zebra_dplane_info info; + + /* Read event */ + struct thread *t_read; + + /* List linkage */ + struct zns_info_list_item link; +}; + /* * Globals */ @@ -424,6 +438,9 @@ static struct zebra_dplane_globals { /* Ordered list of providers */ TAILQ_HEAD(zdg_prov_q, zebra_dplane_provider) dg_providers_q; + /* List of info about each zns */ + struct zns_info_list_head dg_zns_list; + /* Counter used to assign internal ids to providers */ uint32_t dg_provider_id; @@ -498,6 +515,9 @@ static struct zebra_dplane_globals { } zdplane_info; +/* Instantiate zns list type */ +DECLARE_DLIST(zns_info_list, struct dplane_zns_info, link); + /* * Lock and unlock for interactions with the zebra 'core' pthread */ @@ -4813,6 +4833,88 @@ static void dplane_info_from_zns(struct zebra_dplane_info *ns_info, #endif /* NETLINK */ } +#ifdef HAVE_NETLINK +/* + * Callback when an OS (netlink) incoming event read is ready. This runs + * in the dplane pthread. + */ +static int dplane_incoming_read(struct thread *event) +{ + struct dplane_zns_info *zi = THREAD_ARG(event); + + kernel_dplane_read(&zi->info); + + /* Re-start read task */ + thread_add_read(zdplane_info.dg_master, dplane_incoming_read, zi, + zi->info.nls.sock, &zi->t_read); + + return 0; +} +#endif /* HAVE_NETLINK */ + +/* + * Notify dplane when namespaces are enabled and disabled. The dplane + * needs to start and stop reading incoming events from the zns. In the + * common case where vrfs are _not_ namespaces, there will only be one + * of these. + * + * This is called in the main pthread. + */ +void zebra_dplane_ns_enable(struct zebra_ns *zns, bool enabled) +{ + struct dplane_zns_info *zi; + + if (IS_ZEBRA_DEBUG_DPLANE) + zlog_debug("%s: %s for nsid %u", __func__, + (enabled ? "ENABLED" : "DISABLED"), zns->ns_id); + + /* Search for an existing zns info entry */ + frr_each (zns_info_list, &zdplane_info.dg_zns_list, zi) { + if (zi->info.ns_id == zns->ns_id) + break; + } + + if (enabled) { + /* Create a new entry if necessary; start reading. */ + if (zi == NULL) { + zi = XCALLOC(MTYPE_DP_NS, sizeof(*zi)); + + zi->info.ns_id = zns->ns_id; + + zns_info_list_add_tail(&zdplane_info.dg_zns_list, zi); + + if (IS_ZEBRA_DEBUG_DPLANE) + zlog_debug("%s: nsid %u, new zi %p", __func__, + zns->ns_id, zi); + } + + /* Make sure we're up-to-date with the zns object */ +#if defined(HAVE_NETLINK) + zi->info.is_cmd = false; + zi->info.nls = zns->netlink_dplane_in; + + /* Start read task for the dplane pthread. */ + if (zdplane_info.dg_master) + thread_add_read(zdplane_info.dg_master, + dplane_incoming_read, zi, + zi->info.nls.sock, &zi->t_read); +#endif + } else if (zi) { + if (IS_ZEBRA_DEBUG_DPLANE) + zlog_debug("%s: nsid %u, deleting zi %p", __func__, + zns->ns_id, zi); + + /* Stop reading, free memory */ + zns_info_list_del(&zdplane_info.dg_zns_list, zi); + + if (zdplane_info.dg_master) + thread_cancel_async(zdplane_info.dg_master, &zi->t_read, + NULL); + + XFREE(MTYPE_DP_NS, zi); + } +} + /* * Provider api to signal that work/events are available * for the dataplane pthread. @@ -5468,9 +5570,21 @@ done: */ static int dplane_check_shutdown_status(struct thread *event) { + struct dplane_zns_info *zi; + if (IS_ZEBRA_DEBUG_DPLANE) zlog_debug("Zebra dataplane shutdown status check called"); + /* Remove any zns info entries as we stop the dplane pthread. */ + frr_each_safe (zns_info_list, &zdplane_info.dg_zns_list, zi) { + zns_info_list_del(&zdplane_info.dg_zns_list, zi); + + if (zdplane_info.dg_master) + thread_cancel(&zi->t_read); + + XFREE(MTYPE_DP_NS, zi); + } + if (dplane_work_pending()) { /* Reschedule dplane check on a short timer */ thread_add_timer_msec(zdplane_info.dg_master, @@ -5765,6 +5879,7 @@ static void zebra_dplane_init_internal(void) TAILQ_INIT(&zdplane_info.dg_update_ctx_q); TAILQ_INIT(&zdplane_info.dg_providers_q); + zns_info_list_init(&zdplane_info.dg_zns_list); zdplane_info.dg_updates_per_cycle = DPLANE_DEFAULT_NEW_WORK; @@ -5780,6 +5895,7 @@ static void zebra_dplane_init_internal(void) */ void zebra_dplane_start(void) { + struct dplane_zns_info *zi; struct zebra_dplane_provider *prov; struct frr_pthread_attr pattr = { .start = frr_pthread_attr_default.start, @@ -5799,6 +5915,14 @@ void zebra_dplane_start(void) thread_add_event(zdplane_info.dg_master, dplane_thread_loop, NULL, 0, &zdplane_info.dg_t_update); + /* Enqueue reads if necessary */ + frr_each (zns_info_list, &zdplane_info.dg_zns_list, zi) { +#if defined(HAVE_NETLINK) + thread_add_read(zdplane_info.dg_master, dplane_incoming_read, + zi, zi->info.nls.sock, &zi->t_read); +#endif + } + /* Call start callbacks for registered providers */ DPLANE_LOCK(); diff --git a/zebra/zebra_ns.c b/zebra/zebra_ns.c index 27b8a3ea47..8ae677fb22 100644 --- a/zebra/zebra_ns.c +++ b/zebra/zebra_ns.c @@ -123,6 +123,7 @@ int zebra_ns_enable(ns_id_t ns_id, void **info) zns->ns_id = ns_id; kernel_init(zns); + zebra_dplane_ns_enable(zns, true); interface_list(zns); route_read(zns); kernel_read_pbr_rules(zns); @@ -140,6 +141,8 @@ static int zebra_ns_disable_internal(struct zebra_ns *zns, bool complete) { route_table_finish(zns->if_table); + zebra_dplane_ns_enable(zns, false /*Disable*/); + kernel_terminate(zns, complete); table_manager_disable(zns->ns_id); diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index 76327fbc60..24c51e485f 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -4209,6 +4209,11 @@ static int rib_process_dplane_results(struct thread *thread) zebra_pbr_dplane_result(ctx); break; + case DPLANE_OP_INTF_ADDR_ADD: + case DPLANE_OP_INTF_ADDR_DEL: + zebra_if_addr_update_ctx(ctx); + break; + /* Some op codes not handled here */ case DPLANE_OP_ADDR_INSTALL: case DPLANE_OP_ADDR_UNINSTALL: @@ -4223,8 +4228,6 @@ static int rib_process_dplane_results(struct thread *thread) case DPLANE_OP_BR_PORT_UPDATE: case DPLANE_OP_NEIGH_TABLE_UPDATE: case DPLANE_OP_GRE_SET: - case DPLANE_OP_INTF_ADDR_ADD: - case DPLANE_OP_INTF_ADDR_DEL: case DPLANE_OP_NONE: /* Don't expect this: just return the struct? */ dplane_ctx_fini(&ctx); -- 2.39.5