Merge pull request #8673 from LabNConsulting/chopps/ly2-again

lib: libyang2 add missed conversion

8 files changed, 88 insertions, 65 deletions

diff --git a/.travis.yml b/.travis.yml
index 010292bb6a..32b686c00e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,14 +5,14 @@ services:
   - docker
 jobs:
   include:
-   # - script:
-   #     - docker/centos-7/build.sh
-   #     - docker images
-   #   name: centos7
-   # - script:
-   #     - docker/centos-8/build.sh
-   #     - docker images
-   #   name: centos8
+   - script:
+       - docker/centos-7/build.sh
+       - docker images
+     name: centos7
+   - script:
+       - docker/centos-8/build.sh
+       - docker images
+     name: centos8
    - script:
        - sudo apt install -y linux-modules-extra-$(uname -r)
        - docker build -t frr-ubuntu18:latest -f docker/ubuntu18-ci/Dockerfile .
diff --git a/Makefile.am b/Makefile.am
index a5101df2f0..a38029dcfa 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -11,6 +11,11 @@ AM_CFLAGS = \
 	$(SAN_FLAGS) \
 	$(WERROR) \
 	# end
+AM_CXXFLAGS = \
+	$(AC_CXXFLAGS) \
+	$(LIBYANG_CFLAGS) \
+	$(WERROR) \
+	# end
 
 # CPPFLAGS_BASE does not contain the include path for overriding assert.h,
 # therefore should be used in tools that do *not* link libfrr or do not want
diff --git a/doc/developer/building-libyang.rst b/doc/developer/building-libyang.rst
index a447f58309..3b33eb3879 100644
--- a/doc/developer/building-libyang.rst
+++ b/doc/developer/building-libyang.rst
@@ -10,7 +10,7 @@ The FRR project builds some binary ``libyang`` packages.
 RPM packages are at our `RPM repository <https://rpm.frrouting.org>`_.
 
 DEB packages are available as CI artifacts `here
-<https://ci1.netdef.org/browse/LIBYANG-LIBYANG-V2/latestSuccessful/artifact>`_.
+<https://ci1.netdef.org/browse/LIBYANG-LIBYANGV2/latestSuccessful/artifact>`_.
 
 .. warning::
 
diff --git a/docker/centos-7/Dockerfile b/docker/centos-7/Dockerfile
index d2ec9f974b..748b5345a1 100644
--- a/docker/centos-7/Dockerfile
+++ b/docker/centos-7/Dockerfile
@@ -5,8 +5,8 @@ RUN yum install -y rpm-build autoconf automake libtool make \
         readline-devel texinfo net-snmp-devel groff pkgconfig \
         json-c-devel pam-devel bison flex pytest c-ares-devel \
         python3-devel python3-sphinx systemd-devel libcap-devel \
-        https://ci1.netdef.org/artifact/LIBYANG-LY1REL/shared/build-4/CentOS-7-x86_64-Packages/libyang1-1.0.184-0.x86_64.rpm \
-        https://ci1.netdef.org/artifact/LIBYANG-LY1REL/shared/build-4/CentOS-7-x86_64-Packages/libyang-devel-1.0.184-0.x86_64.rpm \
+        https://ci1.netdef.org/artifact/LIBYANG-LIBYANGV2/shared/build-2/CentOS-7-x86_64-Packages/libyang2-2.0.0.10.g2eb910e4-1.el7.x86_64.rpm \
+        https://ci1.netdef.org/artifact/LIBYANG-LIBYANGV2/shared/build-2/CentOS-7-x86_64-Packages/libyang2-devel-2.0.0.10.g2eb910e4-1.el7.x86_64.rpm \
         https://ci1.netdef.org/artifact/RPKI-RTRLIB/shared/build-110/CentOS-7-x86_64-Packages/librtr-0.7.0-1.el7.centos.x86_64.rpm \
         https://ci1.netdef.org/artifact/RPKI-RTRLIB/shared/build-110/CentOS-7-x86_64-Packages/librtr-devel-0.7.0-1.el7.centos.x86_64.rpm
 
@@ -32,7 +32,7 @@ RUN echo '%_smp_mflags %( echo "-j$(/usr/bin/getconf _NPROCESSORS_ONLN)"; )' >>
 # This stage installs frr from the rpm
 FROM centos:centos7
 RUN mkdir -p /pkgs/rpm \
-    && yum install -y https://ci1.netdef.org/artifact/LIBYANG-LY1REL/shared/build-4/CentOS-7-x86_64-Packages/libyang1-1.0.184-0.x86_64.rpm \
+    && yum install -y https://ci1.netdef.org/artifact/LIBYANG-LIBYANGV2/shared/build-2/CentOS-7-x86_64-Packages/libyang2-2.0.0.10.g2eb910e4-1.el7.x86_64.rpm \
         https://ci1.netdef.org/artifact/RPKI-RTRLIB/shared/build-110/CentOS-7-x86_64-Packages/librtr-0.7.0-1.el7.centos.x86_64.rpm
 
 COPY --from=centos-7-builder /rpmbuild/RPMS/ /pkgs/rpm/
diff --git a/docker/centos-8/Dockerfile b/docker/centos-8/Dockerfile
index 104501aabc..e273be055b 100644
--- a/docker/centos-8/Dockerfile
+++ b/docker/centos-8/Dockerfile
@@ -5,8 +5,8 @@ RUN dnf install --enablerepo=powertools -y rpm-build git autoconf pcre-devel \
         automake libtool make readline-devel texinfo net-snmp-devel pkgconfig \
         groff pkgconfig json-c-devel pam-devel bison flex python3-pytest \
         c-ares-devel python3-devel python3-sphinx systemd-devel libcap-devel platform-python-devel \
-        https://ci1.netdef.org/artifact/LIBYANG-LY1REL/shared/build-4/CentOS-8-x86_64-Packages/libyang1-1.0.184-0.x86_64.rpm \
-        https://ci1.netdef.org/artifact/LIBYANG-LY1REL/shared/build-4/CentOS-8-x86_64-Packages/libyang-devel-1.0.184-0.x86_64.rpm \
+        https://ci1.netdef.org/artifact/LIBYANG-LIBYANGV2/shared/build-2/CentOS-8-x86_64-Packages/libyang2-2.0.0.10.g2eb910e4-1.el8.x86_64.rpm \
+        https://ci1.netdef.org/artifact/LIBYANG-LIBYANGV2/shared/build-2/CentOS-8-x86_64-Packages/libyang2-devel-2.0.0.10.g2eb910e4-1.el8.x86_64.rpm \
         https://ci1.netdef.org/artifact/RPKI-RTRLIB/shared/build-110/CentOS-7-x86_64-Packages/librtr-0.7.0-1.el7.centos.x86_64.rpm \
         https://ci1.netdef.org/artifact/RPKI-RTRLIB/shared/build-110/CentOS-7-x86_64-Packages/librtr-devel-0.7.0-1.el7.centos.x86_64.rpm
 
@@ -33,7 +33,7 @@ RUN echo '%_smp_mflags %( echo "-j$(/usr/bin/getconf _NPROCESSORS_ONLN)"; )' >>
 # This stage installs frr from the rpm
 FROM centos:centos8
 RUN mkdir -p /pkgs/rpm \
-    && yum install -y https://ci1.netdef.org/artifact/LIBYANG-LY1REL/shared/build-4/CentOS-8-x86_64-Packages/libyang1-1.0.184-0.x86_64.rpm \
+    && yum install -y https://ci1.netdef.org/artifact/LIBYANG-LIBYANGV2/shared/build-2/CentOS-8-x86_64-Packages/libyang2-2.0.0.10.g2eb910e4-1.el8.x86_64.rpm \
         https://ci1.netdef.org/artifact/RPKI-RTRLIB/shared/build-110/CentOS-7-x86_64-Packages/librtr-0.7.0-1.el7.centos.x86_64.rpm
 
 COPY --from=centos-8-builder /rpmbuild/RPMS/ /pkgs/rpm/
diff --git a/lib/northbound_db.c b/lib/northbound_db.c
index 244e760b2b..dce9b2ec24 100644
--- a/lib/northbound_db.c
+++ b/lib/northbound_db.c
@@ -87,9 +87,12 @@ int nb_db_transaction_save(const struct nb_transaction *transaction,
 		goto exit;
 
 	client_name = nb_client_name(transaction->context->client);
-	/* Always record configurations in the XML format. */
+	/*
+	 * Always record configurations in the XML format, save the default
+	 * values too, as this covers the case where defaults may change.
+	 */
 	if (lyd_print_mem(&config_str, transaction->config->dnode, LYD_XML,
-			  LYP_FORMAT | LYP_WITHSIBLINGS)
+			  LYD_PRINT_WITHSIBLINGS | LYD_PRINT_WD_ALL)
 	    != 0)
 		goto exit;
 
@@ -149,6 +152,7 @@ struct nb_config *nb_db_transaction_load(uint32_t transaction_id)
 	struct lyd_node *dnode;
 	const char *config_str;
 	struct sqlite3_stmt *ss;
+	LY_ERR err;
 
 	ss = db_prepare(
 		"SELECT\n"
@@ -169,10 +173,11 @@ struct nb_config *nb_db_transaction_load(uint32_t transaction_id)
 	if (db_loadf(ss, "%s", &config_str) != 0)
 		goto exit;
 
-	dnode = lyd_parse_mem(ly_native_ctx, config_str, LYD_XML,
-			      LYD_OPT_CONFIG);
-	if (!dnode)
-		flog_warn(EC_LIB_LIBYANG, "%s: lyd_parse_mem() failed",
+	err = lyd_parse_data_mem(ly_native_ctx, config_str, LYD_XML,
+				 LYD_PARSE_STRICT | LYD_PARSE_NO_STATE,
+				 LYD_VALIDATE_NO_STATE, &dnode);
+	if (err || !dnode)
+		flog_warn(EC_LIB_LIBYANG, "%s: lyd_parse_data_mem() failed",
 			  __func__);
 	else
 		config = nb_config_new(dnode);
diff --git a/lib/northbound_grpc.cpp b/lib/northbound_grpc.cpp
index dc2d29c11d..c61effdda5 100644
--- a/lib/northbound_grpc.cpp
+++ b/lib/northbound_grpc.cpp
@@ -199,9 +199,8 @@ class NorthboundImpl
 				auto m = tag->response.add_supported_modules();
 
 				m->set_name(module->name);
-				if (module->info->rev_size)
-					m->set_revision(
-						module->info->rev[0].date);
+				if (module->info->revision)
+					m->set_revision(module->info->revision);
 				m->set_organization(module->info->org);
 			}
 
@@ -1068,14 +1067,13 @@ class NorthboundImpl
 				   const std::string &path,
 				   const std::string &value)
 	{
-		ly_errno = LY_SUCCESS;
-		dnode = lyd_new_path(dnode, ly_native_ctx, path.c_str(),
-				     (void *)value.c_str(),
-				     (LYD_ANYDATA_VALUETYPE)0,
-				     LYD_PATH_OPT_UPDATE);
-		if (!dnode && ly_errno != LY_SUCCESS) {
-			flog_warn(EC_LIB_LIBYANG, "%s: lyd_new_path() failed",
-				  __func__);
+		LY_ERR err = lyd_new_path(dnode, ly_native_ctx, path.c_str(),
+					  value.c_str(), LYD_NEW_PATH_UPDATE,
+					  &dnode);
+		if (err != LY_SUCCESS) {
+			flog_warn(EC_LIB_LIBYANG,
+				  "%s: lyd_new_path() failed: %s", __func__,
+				  ly_errmsg(ly_native_ctx));
 			return -1;
 		}
 
@@ -1089,7 +1087,7 @@ class NorthboundImpl
 		if (!dnode)
 			return -1;
 
-		lyd_free(dnode);
+		lyd_free_tree(dnode);
 
 		return 0;
 	}
@@ -1132,46 +1130,53 @@ class NorthboundImpl
 			std::string(date), std::string(comment)));
 	}
 
-	static int data_tree_from_dnode(frr::DataTree *dt,
-					const struct lyd_node *dnode,
-					LYD_FORMAT lyd_format,
-					bool with_defaults)
+	static LY_ERR data_tree_from_dnode(frr::DataTree *dt,
+					   const struct lyd_node *dnode,
+					   LYD_FORMAT lyd_format,
+					   bool with_defaults)
 	{
 		char *strp;
 		int options = 0;
 
-		SET_FLAG(options, LYP_FORMAT | LYP_WITHSIBLINGS);
+		SET_FLAG(options, LYD_PRINT_WITHSIBLINGS);
 		if (with_defaults)
-			SET_FLAG(options, LYP_WD_ALL);
+			SET_FLAG(options, LYD_PRINT_WD_ALL);
 		else
-			SET_FLAG(options, LYP_WD_TRIM);
+			SET_FLAG(options, LYD_PRINT_WD_TRIM);
 
-		if (lyd_print_mem(&strp, dnode, lyd_format, options) == 0) {
+		LY_ERR err = lyd_print_mem(&strp, dnode, lyd_format, options);
+		if (err == LY_SUCCESS) {
 			if (strp) {
 				dt->set_data(strp);
 				free(strp);
 			}
-			return 0;
 		}
-
-		return -1;
+		return err;
 	}
 
 	static struct lyd_node *dnode_from_data_tree(const frr::DataTree *dt,
 						     bool config_only)
 	{
 		struct lyd_node *dnode;
-		int options;
-
-		if (config_only)
-			options = LYD_OPT_CONFIG;
-		else
-			options = LYD_OPT_DATA | LYD_OPT_DATA_NO_YANGLIB;
-
-		dnode = lyd_parse_mem(ly_native_ctx, dt->data().c_str(),
-				      encoding2lyd_format(dt->encoding()),
-				      options);
+		int options, opt2;
+		LY_ERR err;
+
+		if (config_only) {
+			options = LYD_PARSE_STRICT | LYD_PARSE_NO_STATE;
+			opt2 = LYD_VALIDATE_NO_STATE;
+		} else {
+			options = LYD_PARSE_STRICT;
+			opt2 = 0;
+		}
 
+		err = lyd_parse_data_mem(ly_native_ctx, dt->data().c_str(),
+					 encoding2lyd_format(dt->encoding()),
+					 options, opt2, &dnode);
+		if (err != LY_SUCCESS) {
+			flog_warn(EC_LIB_LIBYANG,
+				  "%s: lyd_parse_mem() failed: %s", __func__,
+				  ly_errmsg(ly_native_ctx));
+		}
 		return dnode;
 	}
 
@@ -1239,14 +1244,15 @@ class NorthboundImpl
 			// Combine configuration and state data into a single
 			// dnode.
 			//
-			if (lyd_merge(dnode_state, dnode_config,
-				      LYD_OPT_EXPLICIT)
-			    != 0) {
+			if (lyd_merge_tree(&dnode_state, dnode_config,
+					   LYD_MERGE_DESTRUCT)
+			    != LY_SUCCESS) {
 				yang_dnode_free(dnode_state);
 				yang_dnode_free(dnode_config);
 				return grpc::Status(
 					grpc::StatusCode::INTERNAL,
-					"Failed to merge configuration and state data");
+					"Failed to merge configuration and state data",
+					ly_errmsg(ly_native_ctx));
 			}
 
 			dnode_final = dnode_state;
@@ -1262,19 +1268,25 @@ class NorthboundImpl
 		// Validate data to create implicit default nodes if necessary.
 		int validate_opts = 0;
 		if (type == frr::GetRequest_DataType_CONFIG)
-			validate_opts = LYD_OPT_CONFIG;
+			validate_opts = LYD_VALIDATE_NO_STATE;
 		else
-			validate_opts = LYD_OPT_DATA | LYD_OPT_DATA_NO_YANGLIB;
-		lyd_validate(&dnode_final, validate_opts, ly_native_ctx);
+			validate_opts = 0;
 
+		LY_ERR err = lyd_validate_all(&dnode_final, ly_native_ctx,
+					      validate_opts, NULL);
+
+		if (err)
+			flog_warn(EC_LIB_LIBYANG,
+				  "%s: lyd_validate_all() failed: %s", __func__,
+				  ly_errmsg(ly_native_ctx));
 		// Dump data using the requested format.
-		int ret = data_tree_from_dnode(dt, dnode_final, lyd_format,
-					       with_defaults);
+		if (!err)
+			err = data_tree_from_dnode(dt, dnode_final, lyd_format,
+						   with_defaults);
 		yang_dnode_free(dnode_final);
-		if (ret != 0)
+		if (err)
 			return grpc::Status(grpc::StatusCode::INTERNAL,
 					    "Failed to dump data");
-
 		return grpc::Status::OK;
 	}
 
diff --git a/lib/xref.h b/lib/xref.h
index 949458b313..6cff1a3769 100644
--- a/lib/xref.h
+++ b/lib/xref.h
@@ -169,6 +169,7 @@ extern const struct xref * const __stop_xref_array[1] DSO_LOCAL;
 	static void __attribute__((used, _CONSTRUCTOR(1100)))                  \
 			_xref_init(void) {                                     \
 		static struct xref_block _xref_block = {                       \
+			.next = NULL,                                          \
 			.start = __start_xref_array,                           \
 			.stop = __stop_xref_array,                             \
 		};                                                             \