build: rewrite readline/curses check

We don't need termcap/tinfo/curses, those are just for libreadline.  On
most modern systems, libreadline will pull in the appropriate libs it
needs on its own, so unconditionally adding them adds an extra unneeded
dependency for us.

Still need to try with curses/... for some systems, but only after bare
readline fails.

Signed-off-by: David Lamparter <equinox@diac24.net>

3 files changed, 43 insertions, 38 deletions

diff --git a/configure.ac b/configure.ac
index fad306a4fb..15bf059c89 100755
--- a/configure.ac
+++ b/configure.ac
@@ -984,7 +984,6 @@ case "$host_os" in
     AC_CHECK_LIB(socket, main)
     AC_CHECK_LIB(nsl, main)
     AC_CHECK_LIB(umem, main)
-    CURSES=-lcurses
     SOLARIS="solaris"
     ;;
   linux*)
@@ -1050,40 +1049,47 @@ dnl ---------------------
 dnl Integrated VTY option
 dnl ---------------------
 case "${enable_vtysh}" in
-  "no") VTYSH="";;
-  *)    VTYSH="vtysh";
-         AC_DEFINE(VTYSH,,VTY shell)
-dnl	 Vtysh uses libreadline, which looks for termcap functions at
-dnl	 configure time.  We follow readlines search order.
-dnl	 The required procedures are in libtermcap on NetBSD, in
-dnl	 [TODO] on Linux, and in [TODO] on Solaris.
-	 AC_CHECK_LIB(termcap, tputs, LIBREADLINE="$LIBREADLINE -ltermcap",
-	   [AC_CHECK_LIB(tinfo, tputs, LIBREADLINE="$LIBREADLINE -ltinfo",
-	     [AC_CHECK_LIB(curses, tputs, LIBREADLINE="$LIBREADLINE -lcurses",
-	       [AC_CHECK_LIB(ncurses, tputs, 
-	                     LIBREADLINE="$LIBREADLINE -lncurses")]
-	      )]
-	    )]
-	  )
-         AC_CHECK_LIB(readline, main, LIBREADLINE="-lreadline $LIBREADLINE",,
-                      "$LIBREADLINE")
-         if test $ac_cv_lib_readline_main = no; then
-           AC_MSG_ERROR([vtysh needs libreadline but was not found and usable on your system.])
-         fi
-	 AC_CHECK_HEADER(readline/history.h)
-	 if test $ac_cv_header_readline_history_h = no;then
-           AC_MSG_ERROR([readline is too old to have readline/history.h, please update to the latest readline library.])
-	 fi
-	 AC_CHECK_LIB(readline, rl_completion_matches, 
-	              LIBREADLINE="$LIBREADLINE",, "$LIBREADLINE")
-         if test $ac_cv_lib_readline_rl_completion_matches = no; then
-           AC_DEFINE(rl_completion_matches,completion_matches,Old readline)
-	 fi
-	 AC_SEARCH_LIBS([append_history], [readline], [frr_cv_append_history=yes], [frr_cv_append_history=no])
-	 if test "$frr_cv_append_history" = yes; then
-	   AC_DEFINE(HAVE_APPEND_HISTORY, 1, [Have history.h append_history])
-	 fi
-	 ;;
+"no")
+  VTYSH="";;
+*)
+  VTYSH="vtysh";
+  AC_DEFINE(VTYSH,,VTY shell)
+
+  prev_libs="$LIBS"
+  AC_CHECK_LIB(readline, main, [
+    LIBREADLINE="-lreadline"
+  ], [
+    dnl readline failed - it might be incorrectly linked and missing its
+    dnl termcap/tinfo/curses dependency.  see if we can fix that...
+    AC_SEARCH_LIBS(tputs, [termcap tinfo curses ncurses], [
+      LIBREADLINE="$ac_cv_search_tputs"
+    ], [
+      AC_MSG_ERROR([libreadline (needed for vtysh) not found and/or missing dependencies])
+    ])
+
+    dnl re-try with the lib we found above
+    unset ac_cv_lib_readline_main
+    AC_CHECK_LIB(readline, main, [
+      LIBREADLINE="-lreadline $LIBREADLINE"
+    ], [
+      AC_MSG_ERROR([libreadline (needed for vtysh) not found and/or missing dependencies])
+    ], [$LIBREADLINE])
+  ], [])
+  LIBS="$prev_libs"
+
+  AC_CHECK_HEADER(readline/history.h)
+  if test $ac_cv_header_readline_history_h = no;then
+    AC_MSG_ERROR([readline is too old to have readline/history.h, please update to the latest readline library.])
+  fi
+  AC_CHECK_LIB(readline, rl_completion_matches, [true], [], [$LIBREADLINE])
+  if test $ac_cv_lib_readline_rl_completion_matches = no; then
+    AC_DEFINE(rl_completion_matches,completion_matches,Old readline)
+  fi
+  AC_CHECK_LIB(readline, [append_history], [frr_cv_append_history=yes], [frr_cv_append_history=no], [$LIBREADLINE])
+  if test "$frr_cv_append_history" = yes; then
+    AC_DEFINE(HAVE_APPEND_HISTORY, 1, [Have history.h append_history])
+  fi
+  ;;
 esac
 AC_SUBST(LIBREADLINE)
 AM_CONDITIONAL(VTYSH, test "x$VTYSH" = "xvtysh")
@@ -1484,7 +1490,6 @@ fi
 AM_CONDITIONAL([ENABLE_BGP_VNC], [test x${enable_bgp_vnc} != xno])
 
 AC_SUBST(SOLARIS)
-AC_SUBST(CURSES)
 AC_CHECK_LIB(crypt, crypt, [],
 		    [AC_CHECK_LIB(crypto, DES_crypt)])
 AC_CHECK_LIB(resolv, res_init)
diff --git a/debianpkg/control b/debianpkg/control
index 67b701e75d..ea977937bf 100644
--- a/debianpkg/control
+++ b/debianpkg/control
@@ -4,7 +4,7 @@ Priority: optional
 Maintainer: Nobody <nobody@frrouting.org>
 Uploaders: Nobody <nobody@frrouting.org>
 XSBC-Original-Maintainer: <maintainers@frrouting.org>
-Build-Depends: debhelper (>= 7.0.50~), libncurses5-dev, libreadline-dev, libpam0g-dev | libpam-dev, libcap-dev, texinfo (>= 4.7), autotools-dev, libpcre3-dev, gawk, chrpath, libsnmp-dev, git, dh-autoreconf, libjson-c-dev, libjson-c2 | libjson-c3, dh-systemd, libsystemd-dev, bison, flex, libc-ares-dev, pkg-config, python (>= 2.7) | python3, python-sphinx | python3-sphinx, libpython-dev | libpython3-dev, install-info
+Build-Depends: debhelper (>= 7.0.50~), libreadline-dev, libpam0g-dev | libpam-dev, libcap-dev, texinfo (>= 4.7), autotools-dev, libpcre3-dev, gawk, chrpath, libsnmp-dev, git, dh-autoreconf, libjson-c-dev, libjson-c2 | libjson-c3, dh-systemd, libsystemd-dev, bison, flex, libc-ares-dev, pkg-config, python (>= 2.7) | python3, python-sphinx | python3-sphinx, libpython-dev | libpython3-dev, install-info
 Standards-Version: 3.9.6
 Homepage: http://www.frrouting.org/
 
diff --git a/vtysh/subdir.am b/vtysh/subdir.am
index 932429a87c..66a31ffff3 100644
--- a/vtysh/subdir.am
+++ b/vtysh/subdir.am
@@ -24,7 +24,7 @@ noinst_HEADERS += \
 	vtysh/vtysh_user.h \
 	# end
 
-vtysh_vtysh_LDADD = lib/libfrr.la @LIBCAP@ @LIBREADLINE@ @LIBS@ @CURSES@ @LIBPAM@
+vtysh_vtysh_LDADD = lib/libfrr.la @LIBCAP@ @LIBREADLINE@ @LIBS@ @LIBPAM@
 
 EXTRA_DIST += vtysh/extract.pl