Switch IDNA implementation to libidn2 [BZ #19728] [BZ #19729] [BZ #22247]

This provides an implementation of the IDNA2008 standard and fixes
CVE-2016-6261, CVE-2016-6263, CVE-2017-14062.

44 files changed, 1351 insertions, 19143 deletions

diff --git a/ChangeLog b/ChangeLog
index ce503a7b85..c2758fdec6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,69 @@
 2018-05-23  Florian Weimer  <fweimer@redhat.com>
 
+	[BZ #19728]
+	[BZ #19729]
+	[BZ #22247]
+	CVE-2016-6261
+	CVE-2016-6263
+	CVE-2017-14062
+	Switch to extern IDNA implementation (libidn2).
+	* libidn: Remove subdirectory.
+	* LICENSES: Do not mention licensing conditions for the removed
+	libidn code.
+	* config.h.in (HAVE_LIBIDN): Remove.
+	* include/dlfcn.h (__libc_dlopen): Update comment.
+	* include/idna.h: Remove file.
+	* inet/Makefile (routines): Add idna.
+	(tests-static, tests-internal): Add tst-idna_name_classify.
+	(LOCALES): Generate locales for tests.
+	(tst-idna_name_classify.out): Depend on generated locales.
+	* inet/idna_name_classify.c: New file.
+	* inet/tst-idna_name_classify.c: Likewise.
+	* inet/net-internal.h (__idna_to_dns_encoding)
+	(__idna_from_dns_encoding): Declare.
+	* inet/net-internal.h (enum idna_name_classification): Define.
+	(__idna_name_classify): Declare.
+	* inet/Versions (GLIBC_PRIVATE): Add __idna_to_dns_encoding,
+	__idna_from_dns_encoding.
+	* inet/getnameinfo.c (DEPRECATED_NI_IDN): Define.
+	(gni_host_inet_name): Call __idna_from_dns_encoding.  Use punycode
+	name as a fallback in case of encoding errors.
+	(getnameinfo): Use DEPRECATED_NI_IDN.
+	* inet/idna.c: New file.
+	* nscd/gai.c: Do not include <libidn/idn-stub.c>.
+	* resolv/Makefile (tests): Add tst-resolv-ai_idn,
+	tst-resolv-ai_idn-latin1, tst-resolv-ai_idn-nolibidn2.
+	(modules-names): Add tst-no-libidn2.
+	(extra-test-objs): Add tst-no-libidn2.os.
+	(LDFLAGS-tst-no-libidn2.so): Set soname.
+	(LOCALES): Set, and generate locales.
+	(tst-resolv-ai_idn): Link with -ldl -lresolv -lpthread.
+	(tst-resolv-ai_idn-latin1): Likewise.
+	(tst-resolv-ai_idn-nolibidn2): Likewise.
+	(tst-resolv-ai_idn.out): Depend on locales.
+	(tst-resolv-ai_idn-latin1.out): Depend on locales.
+	(tst-resolv-ai_idn-nolibidn2.out): Depend on locales and
+	tst-no-libidn2.so.
+	* resolv/netdb.h (AI_IDN_ALLOW_UNASSIGNED)
+	(AI_IDN_USE_STD3_ASCII_RULES, NI_IDN_ALLOW_UNASSIGNED)
+	(NI_IDN_USE_STD3_ASCII_RULES): Deprecate.
+	* resolv/tst-resolv-ai_idn.c: New file.
+	* resolv/tst-resolv-ai_idn-latin1.c: Likewise.
+	* resolv/tst-resolv-ai_idn-nolibidn2.c: Likewise.
+	* resolv/tst-no-libidn2.c: Likewise.
+	* support/support_format_addrinfo.c (format_ai_flags): Do not
+	handle AI_IDN_ALLOW_UNASSIGNED, AI_IDN_USE_STD3_ASCII_RULES.
+	* sysdeps/posix/getaddrinfo.c (DEPRECATED_AI_IDN): Define.
+	(gaih_inet): Call __idna_to_dns_encoding and
+	__idna_from_dns_encoding, and use the original (punycode) name if
+	__idna_from_dns_encoding fails due to an encoding error.
+	(getaddrinfo): Use DEPRECATED_AI_IDN.
+	* sysdeps/unix/inet/Subdirs (libidn): Remove.
+	* sysdeps/unix/inet/configure: Remove file.
+	* sysdeps/unix/inet/configure.ac: Likewise.
+
+2018-05-23  Florian Weimer  <fweimer@redhat.com>
+
 	Implement allocate_once.
 	* include/allocate_once.h: New file.
 	* misc/allocate_once.c: Likewise.
diff --git a/LICENSES b/LICENSES
index 80f7f14879..b29efe0108 100644
--- a/LICENSES
+++ b/LICENSES
@@ -248,75 +248,6 @@ Public License, version 2.1 or later - see the file COPYING.LIB for details.
 If you did not receive a copy of the license with this program, please
 see <http://www.gnu.org/licenses/> to obtain a copy.
 
-The libidn code is copyright Simon Josefsson, with portions copyright
-The Internet Society, Tom Tromey and Red Hat, Inc.:
-
-Copyright (C) 2002, 2003, 2004, 2011  Simon Josefsson
-
-This file is part of GNU Libidn.
-
-GNU Libidn is free software; you can redistribute it and/or
-modify it under the terms of the GNU Lesser General Public
-License as published by the Free Software Foundation; either
-version 2.1 of the License, or (at your option) any later version.
-
-GNU Libidn is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with GNU Libidn; if not, see <http://www.gnu.org/licenses/>.
-
-The following notice applies to portions of libidn/nfkc.c:
-
-This file contains functions from GLIB, including gutf8.c and
-gunidecomp.c, all licensed under LGPL and copyright hold by:
-
-Copyright (C) 1999, 2000 Tom Tromey
-Copyright 2000 Red Hat, Inc.
-
-The following applies to portions of libidn/punycode.c and
-libidn/punycode.h:
-
-This file is derived from RFC 3492bis written by Adam M. Costello.
-
-Disclaimer and license: Regarding this entire document or any
-portion of it (including the pseudocode and C code), the author
-makes no guarantees and is not responsible for any damage resulting
-from its use.  The author grants irrevocable permission to anyone
-to use, modify, and distribute it in any way that does not diminish
-the rights of anyone else to use, modify, and distribute it,
-provided that redistributed derivative works do not contain
-misleading author or version information.  Derivative works need
-not be licensed under similar terms.
-
-Copyright (C) The Internet Society (2003).  All Rights Reserved.
-
-This document and translations of it may be copied and furnished to
-others, and derivative works that comment on or otherwise explain it
-or assist in its implementation may be prepared, copied, published
-and distributed, in whole or in part, without restriction of any
-kind, provided that the above copyright notice and this paragraph are
-included on all such copies and derivative works.  However, this
-document itself may not be modified in any way, such as by removing
-the copyright notice or references to the Internet Society or other
-Internet organizations, except as needed for the purpose of
-developing Internet standards in which case the procedures for
-copyrights defined in the Internet Standards process must be
-followed, or as required to translate it into languages other than
-English.
-
-The limited permissions granted above are perpetual and will not be
-revoked by the Internet Society or its successors or assigns.
-
-This document and the information contained herein is provided on an
-"AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING
-TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING
-BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION
-HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF
-MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE.
-
 The file inet/rcmd.c is under a UCB copyright and the following:
 
 Copyright (C) 1998 WIDE Project.
diff --git a/NEWS b/NEWS
index 7838f79ab2..1577f4310f 100644
--- a/NEWS
+++ b/NEWS
@@ -31,6 +31,16 @@ Major new features:
 * Building and running on GNU/Hurd systems now works without out-of-tree
   patches.
 
+* IDN domain names in getaddrinfo and getnameinfo now use the system libidn2
+  library if installed.  libidn2 version 2.0.5 or later is recommended.  If
+  libidn2 is not available, internationalized domain names are not encoded
+  or decoded even if the AI_IDN or NI_IDN flags are passed to getaddrinfo or
+  getnameinfo.  (getaddrinfo calls with non-ASCII names and AI_IDN will fail
+  with an encoding error.)  Flags which used to change the IDN encoding and
+  decoding behavior (AI_IDN_ALLOW_UNASSIGNED, AI_IDN_USE_STD3_ASCII_RULES,
+  NI_IDN_ALLOW_UNASSIGNED, NI_IDN_USE_STD3_ASCII_RULES) have been
+  deprecated.  They no longer have any effect.
+
 Deprecated and removed features, and other changes affecting compatibility:
 
 * The nonstandard header files <libio.h> and <_G_config.h> are no longer
@@ -79,13 +89,25 @@ Deprecated and removed features, and other changes affecting compatibility:
   and was not declared in a header.  Programs should use the lseek64 name
   for this function instead.
 
+* The AI_IDN_ALLOW_UNASSIGNED and NI_IDN_ALLOW_UNASSIGNED flags for the
+  getaddrinfo and getnameinfo functions have been deprecated.  The behavior
+  previously selected by them is now always enabled.
+
+* The AI_IDN_USE_STD3_ASCII_RULES and NI_IDN_USE_STD3_ASCII_RULES flags for
+  the getaddrinfo and getnameinfo functions have been deprecated.  The STD3
+  restriction (rejecting '_' in host names, among other things) has been
+  removed, for increased compatibility with non-IDN name resolution.
+
 Changes to build and runtime requirements:
 
   [Add changes to build and runtime requirements here]
 
 Security related changes:
 
-  [Add security related changes here]
+  CVE-2016-6261, CVE-2016-6263, CVE-2017-14062: Various vulnerabilities have
+  been fixed by removing the glibc-internal IDNA implementation and using
+  the system-provided libidn2 library instead.  Originally reported by Hanno
+  Böck and Christian Weisgerber.
 
 The following bugs are resolved with this release:
 
diff --git a/config.h.in b/config.h.in
index b0b7cf26cb..141db213a9 100644
--- a/config.h.in
+++ b/config.h.in
@@ -124,9 +124,6 @@
 /* Mach/i386 specific: define if the `i386_set_gdt' RPC is available.  */
 #undef	HAVE_I386_SET_GDT
 
-/* Defined of libidn is available.  */
-#undef HAVE_LIBIDN
-
 /* Define if inlined system calls are available.  */
 #undef HAVE_INLINED_SYSCALLS
 
diff --git a/include/dlfcn.h b/include/dlfcn.h
index 694e71c83b..c231309083 100644
--- a/include/dlfcn.h
+++ b/include/dlfcn.h
@@ -46,7 +46,7 @@ extern char **__libc_argv attribute_hidden;
 
    The use of RTLD_NOW also impacts gconv module loading, backtracing
    (where the unwinder form libgcc_s.so is used), and IDNA functions
-   (which load libidn), all of which load their respective DSOs on
+   (which load libidn2), all of which load their respective DSOs on
    demand, and so should not impact program startup.  That is to say
    that the DSOs are loaded as part of an API call and therefore we
    will be calling that family of API functions shortly so RTLD_NOW or
diff --git a/include/idna.h b/include/idna.h
deleted file mode 100644
index dcb271d575..0000000000
--- a/include/idna.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef _IDNA_H
-#include <libidn/idna.h>
-
-extern __typeof (idna_to_ascii_lz) __idna_to_ascii_lz attribute_hidden;
-extern __typeof (idna_to_unicode_lzlz ) __idna_to_unicode_lzlz
-     attribute_hidden;
-
-#endif
diff --git a/inet/Makefile b/inet/Makefile
index 5d02c37626..09f5ba78fc 100644
--- a/inet/Makefile
+++ b/inet/Makefile
@@ -45,7 +45,7 @@ routines := htonl htons		\
 	    in6_addr getnameinfo if_index ifaddrs inet6_option \
 	    getipv4sourcefilter setipv4sourcefilter \
 	    getsourcefilter setsourcefilter inet6_opt inet6_rth \
-	    inet6_scopeid_pton deadline
+	    inet6_scopeid_pton deadline idna idna_name_classify
 
 aux := check_pf check_native ifreq
 
@@ -59,12 +59,20 @@ tests := htontest test_ifindex tst-ntoa tst-ether_aton tst-network \
 tests-static += tst-deadline
 tests-internal += tst-deadline
 
+# tst-idna_name_classify must be linked statically because it tests
+# internal functionality.
+tests-static += tst-idna_name_classify
+tests-internal += tst-idna_name_classify
+
 # tst-inet6_scopeid_pton also needs internal functions but does not
 # need to be linked statically.
 tests-internal += tst-inet6_scopeid_pton
 
 include ../Rules
 
+LOCALES := en_US.UTF-8 en_US.ISO-8859-1
+include ../gen-locales.mk
+
 ifeq ($(have-thread-library),yes)
 
 CFLAGS-gethstbyad_r.c += -fexceptions
@@ -103,3 +111,5 @@ endif
 ifeq ($(build-static-nss),yes)
 CFLAGS += -DSTATIC_NSS
 endif
+
+$(objpfx)tst-idna_name_classify.out: $(gen-locales)
diff --git a/inet/Versions b/inet/Versions
index 6f663f3648..9b3661e046 100644
--- a/inet/Versions
+++ b/inet/Versions
@@ -88,5 +88,7 @@ libc {
 
     # Used from nscd.
     __inet6_scopeid_pton;
+    __idna_to_dns_encoding;
+    __idna_from_dns_encoding;
   }
 }
diff --git a/inet/getnameinfo.c b/inet/getnameinfo.c
index a20d20b7cd..5d4978e383 100644
--- a/inet/getnameinfo.c
+++ b/inet/getnameinfo.c
@@ -71,10 +71,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <sys/utsname.h>
 #include <libc-lock.h>
 #include <scratch_buffer.h>
-
-#ifdef HAVE_LIBIDN
-# include <idna.h>
-#endif
+#include <net-internal.h>
 
 #ifndef min
 # define min(x,y) (((x) > (y)) ? (y) : (x))
@@ -82,6 +79,9 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 libc_freeres_ptr (static char *domain);
 
+/* Former NI_IDN_ALLOW_UNASSIGNED, NI_IDN_USE_STD3_ASCII_RULES flags,
+   now ignored.  */
+#define DEPRECATED_NI_IDN 192
 
 static char *
 nrl_domainname (void)
@@ -285,41 +285,28 @@ gni_host_inet_name (struct scratch_buffer *tmpbuf,
 	/* Terminate the string after the prefix.  */
 	*c = '\0';
 
-#ifdef HAVE_LIBIDN
       /* If requested, convert from the IDN format.  */
-      if (flags & NI_IDN)
+      bool do_idn = flags & NI_IDN;
+      char *h_name;
+      if (do_idn)
 	{
-	  int idn_flags = 0;
-	  if  (flags & NI_IDN_ALLOW_UNASSIGNED)
-	    idn_flags |= IDNA_ALLOW_UNASSIGNED;
-	  if (flags & NI_IDN_USE_STD3_ASCII_RULES)
-	    idn_flags |= IDNA_USE_STD3_ASCII_RULES;
-
-	  char *out;
-	  int rc = __idna_to_unicode_lzlz (h->h_name, &out,
-					   idn_flags);
-	  if (rc != IDNA_SUCCESS)
-	    {
-	      if (rc == IDNA_MALLOC_ERROR)
-		return EAI_MEMORY;
-	      if (rc == IDNA_DLOPEN_ERROR)
-		return EAI_SYSTEM;
-	      return EAI_IDN_ENCODE;
-	    }
-
-	  if (out != h->