x86-64: Add FMA multiarch functions to libm

This patch adds multiarch functions optimized with -mfma -mavx2 to libm.
e_pow-fma.c is compiled with $(config-cflags-nofma) due to PR 19003.

	* sysdeps/x86_64/fpu/multiarch/Makefile (libm-sysdep_routines):
	Add e_exp-fma, e_log-fma, e_pow-fma, s_atan-fma, e_asin-fma,
	e_atan2-fma, s_sin-fma, s_tan-fma, mplog-fma, mpa-fma,
	slowexp-fma, slowpow-fma, sincos32-fma, doasin-fma, dosincos-fma,
	halfulp-fma, mpexp-fma, mpatan2-fma, mpatan-fma, mpsqrt-fma,
	and mptan-fma.
	(CFLAGS-doasin-fma.c): New.
	(CFLAGS-dosincos-fma.c): Likewise.
	(CFLAGS-e_asin-fma.c): Likewise.
	(CFLAGS-e_atan2-fma.c): Likewise.
	(CFLAGS-e_exp-fma.c): Likewise.
	(CFLAGS-e_log-fma.c): Likewise.
	(CFLAGS-e_pow-fma.c): Likewise.
	(CFLAGS-halfulp-fma.c): Likewise.
	(CFLAGS-mpa-fma.c): Likewise.
	(CFLAGS-mpatan-fma.c): Likewise.
	(CFLAGS-mpatan2-fma.c): Likewise.
	(CFLAGS-mpexp-fma.c): Likewise.
	(CFLAGS-mplog-fma.c): Likewise.
	(CFLAGS-mpsqrt-fma.c): Likewise.
	(CFLAGS-mptan-fma.c): Likewise.
	(CFLAGS-s_atan-fma.c): Likewise.
	(CFLAGS-sincos32-fma.c): Likewise.
	(CFLAGS-slowexp-fma.c): Likewise.
	(CFLAGS-slowpow-fma.c): Likewise.
	(CFLAGS-s_sin-fma.c): Likewise.
	(CFLAGS-s_tan-fma.c): Likewise.
	* sysdeps/x86_64/fpu/multiarch/doasin-fma.c: New file.
	* sysdeps/x86_64/fpu/multiarch/dosincos-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_asin-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_atan2-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_exp-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_log-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_pow-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/halfulp-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/ifunc-avx-fma4.h: Likewise.
	* sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mpa-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mpatan-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mpatan2-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mpexp-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mplog-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mpsqrt-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/mptan-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/s_atan-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/s_sin-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/s_tan-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/sincos32-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/slowexp-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/slowpow-fma.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_asin.c: Rewrite.
	* sysdeps/x86_64/fpu/multiarch/e_atan2.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_exp.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_log.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/e_pow.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/s_atan.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/s_sin.c: Likewise.
	* sysdeps/x86_64/fpu/multiarch/s_tan.c: Likewise.

33 files changed, 554 insertions, 105 deletions

diff --git a/ChangeLog b/ChangeLog
index 7cee945879..5caffcd51e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,64 @@
+2017-08-07  H.J. Lu  <hongjiu.lu@intel.com>
+
+	* sysdeps/x86_64/fpu/multiarch/Makefile (libm-sysdep_routines):
+	Add e_exp-fma, e_log-fma, e_pow-fma, s_atan-fma, e_asin-fma,
+	e_atan2-fma, s_sin-fma, s_tan-fma, mplog-fma, mpa-fma,
+	slowexp-fma, slowpow-fma, sincos32-fma, doasin-fma, dosincos-fma,
+	halfulp-fma, mpexp-fma, mpatan2-fma, mpatan-fma, mpsqrt-fma,
+	and mptan-fma.
+	(CFLAGS-doasin-fma.c): New.
+	(CFLAGS-dosincos-fma.c): Likewise.
+	(CFLAGS-e_asin-fma.c): Likewise.
+	(CFLAGS-e_atan2-fma.c): Likewise.
+	(CFLAGS-e_exp-fma.c): Likewise.
+	(CFLAGS-e_log-fma.c): Likewise.
+	(CFLAGS-e_pow-fma.c): Likewise.
+	(CFLAGS-halfulp-fma.c): Likewise.
+	(CFLAGS-mpa-fma.c): Likewise.
+	(CFLAGS-mpatan-fma.c): Likewise.
+	(CFLAGS-mpatan2-fma.c): Likewise.
+	(CFLAGS-mpexp-fma.c): Likewise.
+	(CFLAGS-mplog-fma.c): Likewise.
+	(CFLAGS-mpsqrt-fma.c): Likewise.
+	(CFLAGS-mptan-fma.c): Likewise.
+	(CFLAGS-s_atan-fma.c): Likewise.
+	(CFLAGS-sincos32-fma.c): Likewise.
+	(CFLAGS-slowexp-fma.c): Likewise.
+	(CFLAGS-slowpow-fma.c): Likewise.
+	(CFLAGS-s_sin-fma.c): Likewise.
+	(CFLAGS-s_tan-fma.c): Likewise.
+	* sysdeps/x86_64/fpu/multiarch/doasin-fma.c: New file.
+	* sysdeps/x86_64/fpu/multiarch/dosincos-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_asin-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_atan2-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_exp-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_log-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_pow-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/halfulp-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/ifunc-avx-fma4.h: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/ifunc-fma4.h: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mpa-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mpatan-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mpatan2-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mpexp-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mplog-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mpsqrt-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/mptan-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/s_atan-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/s_sin-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/s_tan-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/sincos32-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/slowexp-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/slowpow-fma.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_asin.c: Rewrite.
+	* sysdeps/x86_64/fpu/multiarch/e_atan2.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_exp.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_log.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/e_pow.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/s_atan.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/s_sin.c: Likewise.
+	* sysdeps/x86_64/fpu/multiarch/s_tan.c: Likewise.
+
 2017-08-04  Joseph Myers  <joseph@codesourcery.com>
 
 	* sysdeps/generic/math_private.h (__EXPR_FLT128): Remove macro.
diff --git a/sysdeps/x86_64/fpu/multiarch/Makefile b/sysdeps/x86_64/fpu/multiarch/Makefile
index f9ceb09a4e..9daf2cf205 100644
--- a/sysdeps/x86_64/fpu/multiarch/Makefile
+++ b/sysdeps/x86_64/fpu/multiarch/Makefile
@@ -6,6 +6,35 @@ libm-sysdep_routines += s_ceil-sse4_1 s_ceilf-sse4_1 s_floor-sse4_1 \
 			s_floorf-sse4_1 s_nearbyint-sse4_1 \
 			s_nearbyintf-sse4_1 s_rint-sse4_1 s_rintf-sse4_1
 
+libm-sysdep_routines += e_exp-fma e_log-fma e_pow-fma s_atan-fma \
+			e_asin-fma e_atan2-fma s_sin-fma s_tan-fma \
+			mplog-fma mpa-fma slowexp-fma slowpow-fma \
+			sincos32-fma doasin-fma dosincos-fma \
+			halfulp-fma mpexp-fma \
+			mpatan2-fma mpatan-fma mpsqrt-fma mptan-fma
+
+CFLAGS-doasin-fma.c = -mfma -mavx2
+CFLAGS-dosincos-fma.c = -mfma -mavx2
+CFLAGS-e_asin-fma.c = -mfma -mavx2
+CFLAGS-e_atan2-fma.c = -mfma -mavx2
+CFLAGS-e_exp-fma.c = -mfma -mavx2
+CFLAGS-e_log-fma.c = -mfma -mavx2
+CFLAGS-e_pow-fma.c = -mfma -mavx2 $(config-cflags-nofma)
+CFLAGS-halfulp-fma.c = -mfma -mavx2
+CFLAGS-mpa-fma.c = -mfma -mavx2
+CFLAGS-mpatan-fma.c = -mfma -mavx2
+CFLAGS-mpatan2-fma.c = -mfma -mavx2
+CFLAGS-mpexp-fma.c = -mfma -mavx2
+CFLAGS-mplog-fma.c = -mfma -mavx2
+CFLAGS-mpsqrt-fma.c = -mfma -mavx2
+CFLAGS-mptan-fma.c = -mfma -mavx2
+CFLAGS-s_atan-fma.c = -mfma -mavx2
+CFLAGS-sincos32-fma.c = -mfma -mavx2
+CFLAGS-slowexp-fma.c = -mfma -mavx2
+CFLAGS-slowpow-fma.c = -mfma -mavx2
+CFLAGS-s_sin-fma.c = -mfma -mavx2
+CFLAGS-s_tan-fma.c = -mfma -mavx2
+
 libm-sysdep_routines += e_exp-fma4 e_log-fma4 e_pow-fma4 s_atan-fma4 \
 			e_asin-fma4 e_atan2-fma4 s_sin-fma4 s_tan-fma4 \
 			mplog-fma4 mpa-fma4 slowexp-fma4 slowpow-fma4 \
diff --git a/sysdeps/x86_64/fpu/multiarch/doasin-fma.c b/sysdeps/x86_64/fpu/multiarch/doasin-fma.c
new file mode 100644
index 0000000000..7a09865fca
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/doasin-fma.c
@@ -0,0 +1,4 @@
+#define __doasin __doasin_fma
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64/doasin.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/dosincos-fma.c b/sysdeps/x86_64/fpu/multiarch/dosincos-fma.c
new file mode 100644
index 0000000000..5744586bdb
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/dosincos-fma.c
@@ -0,0 +1,6 @@
+#define __docos __docos_fma
+#define __dubcos __dubcos_fma
+#define __dubsin __dubsin_fma
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64/dosincos.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_asin-fma.c b/sysdeps/x86_64/fpu/multiarch/e_asin-fma.c
new file mode 100644
index 0000000000..50e9c64247
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/e_asin-fma.c
@@ -0,0 +1,11 @@
+#define __ieee754_acos __ieee754_acos_fma
+#define __ieee754_asin __ieee754_asin_fma
+#define __cos32 __cos32_fma
+#define __doasin __doasin_fma
+#define __docos __docos_fma
+#define __dubcos __dubcos_fma
+#define __dubsin __dubsin_fma
+#define __sin32 __sin32_fma
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64/e_asin.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_asin.c b/sysdeps/x86_64/fpu/multiarch/e_asin.c
index 111a5b99bd..37a44b2388 100644
--- a/sysdeps/x86_64/fpu/multiarch/e_asin.c
+++ b/sysdeps/x86_64/fpu/multiarch/e_asin.c
@@ -1,26 +1,40 @@
-#include <init-arch.h>
-#include <math.h>
-#include <math_private.h>
-
-extern double __ieee754_acos_sse2 (double);
-extern double __ieee754_asin_sse2 (double);
-extern double __ieee754_acos_fma4 (double);
-extern double __ieee754_asin_fma4 (double);
-
-libm_ifunc (__ieee754_acos,
-	    HAS_ARCH_FEATURE (FMA4_Usable)
-	    ? __ieee754_acos_fma4
-	    : __ieee754_acos_sse2);
-strong_alias (__ieee754_acos, __acos_finite)
+/* Multiple versions of IEEE 754 asin and acos.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+extern double __redirect_ieee754_asin (double);
+extern double __redirect_ieee754_acos (double);
+
+#define SYMBOL_NAME ieee754_asin
+#include "ifunc-fma4.h"
 
-libm_ifunc (__ieee754_asin,
-	    HAS_ARCH_FEATURE (FMA4_Usable)
-	    ? __ieee754_asin_fma4
-	    : __ieee754_asin_sse2);
+libc_ifunc_redirected (__redirect_ieee754_asin, __ieee754_asin,
+		       IFUNC_SELECTOR ());
 strong_alias (__ieee754_asin, __asin_finite)
 
-#define __ieee754_acos __ieee754_acos_sse2
-#define __ieee754_asin __ieee754_asin_sse2
+#undef SYMBOL_NAME
+#define SYMBOL_NAME ieee754_acos
+#include "ifunc-fma4.h"
+
+libc_ifunc_redirected (__redirect_ieee754_acos, __ieee754_acos,
+		       IFUNC_SELECTOR ());
+strong_alias (__ieee754_acos, __acos_finite)
 
 
+#define __ieee754_acos __ieee754_acos_sse2
+#define __ieee754_asin __ieee754_asin_sse2
 #include <sysdeps/ieee754/dbl-64/e_asin.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_atan2-fma.c b/sysdeps/x86_64/fpu/multiarch/e_atan2-fma.c
new file mode 100644
index 0000000000..caba686496
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/e_atan2-fma.c
@@ -0,0 +1,10 @@
+#define __ieee754_atan2 __ieee754_atan2_fma
+#define __add __add_fma
+#define __dbl_mp __dbl_mp_fma
+#define __dvd __dvd_fma
+#define __mpatan2 __mpatan2_fma
+#define __mul __mul_fma
+#define __sub __sub_fma
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64/e_atan2.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_atan2.c b/sysdeps/x86_64/fpu/multiarch/e_atan2.c
index 9ca3c02a44..a2c8cfc159 100644
--- a/sysdeps/x86_64/fpu/multiarch/e_atan2.c
+++ b/sysdeps/x86_64/fpu/multiarch/e_atan2.c
@@ -1,18 +1,29 @@
-#include <init-arch.h>
-#include <math.h>
-#include <math_private.h>
+/* Multiple versions of IEEE 754 atan.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
 
-extern double __ieee754_atan2_sse2 (double, double);
-extern double __ieee754_atan2_avx (double, double);
-extern double __ieee754_atan2_fma4 (double, double);
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
 
-libm_ifunc (__ieee754_atan2,
-	    HAS_ARCH_FEATURE (FMA4_Usable) ? __ieee754_atan2_fma4
-	    : (HAS_ARCH_FEATURE (AVX_Usable)
-	       ? __ieee754_atan2_avx : __ieee754_atan2_sse2));
-strong_alias (__ieee754_atan2, __atan2_finite)
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
 
-#define __ieee754_atan2 __ieee754_atan2_sse2
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+extern double __redirect_ieee754_atan2 (double, double);
 
+#define SYMBOL_NAME ieee754_atan2
+#include "ifunc-avx-fma4.h"
 
+libc_ifunc_redirected (__redirect_ieee754_atan2,
+		       __ieee754_atan2, IFUNC_SELECTOR ());
+strong_alias (__ieee754_atan2, __atan2_finite)
+
+#define __ieee754_atan2 __ieee754_atan2_sse2
 #include <sysdeps/ieee754/dbl-64/e_atan2.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_exp-fma.c b/sysdeps/x86_64/fpu/multiarch/e_exp-fma.c
new file mode 100644
index 0000000000..6e0fdb7941
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/e_exp-fma.c
@@ -0,0 +1,6 @@
+#define __ieee754_exp __ieee754_exp_fma
+#define __exp1 __exp1_fma
+#define __slowexp __slowexp_fma
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64/e_exp.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_exp.c b/sysdeps/x86_64/fpu/multiarch/e_exp.c
index b7d7b5ff27..81211f4ace 100644
--- a/sysdeps/x86_64/fpu/multiarch/e_exp.c
+++ b/sysdeps/x86_64/fpu/multiarch/e_exp.c
@@ -1,18 +1,29 @@
-#include <init-arch.h>
-#include <math.h>
-#include <math_private.h>
+/* Multiple versions of IEEE 754 exp.
+   Copyright (C) 2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
 
-extern double __ieee754_exp_sse2 (double);
-extern double __ieee754_exp_avx (double);
-extern double __ieee754_exp_fma4 (double);
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
 
-libm_ifunc (__ieee754_exp,
-	    HAS_ARCH_FEATURE (FMA4_Usable) ? __ieee754_exp_fma4
-	    : (HAS_ARCH_FEATURE (AVX_Usable)
-	       ? __ieee754_exp_avx : __ieee754_exp_sse2));
-strong_alias (__ieee754_exp, __exp_finite)
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
 
-#define __ieee754_exp __ieee754_exp_sse2
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+extern double __redirect_ieee754_exp (double);
 
+#define SYMBOL_NAME ieee754_exp
+#include "ifunc-avx-fma4.h"
 
+libc_ifunc_redirected (__redirect_ieee754_exp, __ieee754_exp,
+		       IFUNC_SELECTOR ());
+strong_alias (__ieee754_exp, __exp_finite)
+
+#define __ieee754_exp __ieee754_exp_sse2
 #include <sysdeps/ieee754/dbl-64/e_exp.c>
diff --git a/sysdeps/x86_64/fpu/multiarch/e_log-fma.c b/sysdeps/x86_64/fpu/multiarch/e_log-fma.c
new file mode 100644
index 0000000000..a7123b1a06
--- /dev/null
+++ b/sysdeps/x86_64/fpu/multiarch/e_log-fma.c
@@ -0,0 +1,8 @@
+#define __ieee754_log __ieee754_log_fma
+#define __mplog __mplog_fma
+#define __add __add_fma
+#define __dbl_mp __dbl_mp_fma
+#define __sub __sub_fma
+#define SECTION __attribute__ ((section (".text.fma")))
+
+#include <sysdeps/ieee754/dbl-64