Implement fma in soft-fp.

21 files changed, 681 insertions, 109 deletions

diff --git a/ChangeLog b/ChangeLog
index 406ca28bd2..4a2d19ad03 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,54 @@
+2013-07-02  Joseph Myers  <joseph@codesourcery.com>
+
+	[BZ #13304]
+	* soft-fp/op-common.h (_FP_FMA): New macro.
+	* soft-fp/op-1.h (_FP_FRAC_HIGHBIT_DW_1): New macro.
+	(_FP_MUL_MEAT_DW_1_imm): Likewise.  Split out of ...
+	(_FP_MUL_MEAT_1_imm): ... here.
+	(_FP_MUL_MEAT_DW_1_wide): New macro.  Split out of ...
+	(_FP_MUL_MEAT_1_wide): ... here.
+	(_FP_MUL_MEAT_DW_1_hard): Likewise.  Split out of ...
+	(_FP_MUL_MEAT_1_hard): ... here.
+	* soft-fp/op-2.h (_FP_FRAC_HIGHBIT_DW_2): New macro.
+	(_FP_MUL_MEAT_DW_2_wide): Likewise.  Split out of ...
+	(_FP_MUL_MEAT_2_wide): ... here.
+	(_FP_MUL_MEAT_DW_2_wide_3mul): New macro.  Split out of ...
+	(_FP_MUL_MEAT_2_wide_3mul): ... here.
+	(_FP_MUL_MEAT_DW_2_gmp): New macro.  Split out of ...
+	(_FP_MUL_MEAT_2_gmp): ... here.
+	* soft-fp/op-4.h (_FP_FRAC_HIGHBIT_DW_4): New macro.
+	(_FP_MUL_MEAT_DW_4_wide): Likewise.  Split out of ...
+	(_FP_MUL_MEAT_4_wide): ... here.
+	(_FP_MUL_MEAT_DW_4_gmp): New macro.  Split out of ...
+	(_FP_MUL_MEAT_4_gmp): ... here.
+	* soft-fp/single.h (_FP_FRACTBITS_DW_S): New macro.
+	(_FP_WFRACBITS_DW_S): Likewise.
+	(_FP_WFRACXBITS_DW_S): Likewise.
+	(_FP_HIGHBIT_DW_S): Likewise.
+	(FP_FMA_S): Likewise.
+	(_FP_FRAC_HIGH_DW_S): Likewise.
+	* soft-fp/double.h (_FP_FRACTBITS_DW_D): New macro.
+	(_FP_WFRACBITS_DW_D): Likewise.
+	(_FP_WFRACXBITS_DW_D): Likewise.
+	(_FP_HIGHBIT_DW_D): Likewise.
+	(FP_FMA_D): Likewise.
+	(_FP_FRAC_HIGH_DW_D): Likewise.
+	* soft-fp/extended.h (_FP_FRACTBITS_DW_E): New macro.
+	(_FP_WFRACBITS_DW_E): Likewise.
+	(_FP_WFRACXBITS_DW_E): Likewise.
+	(_FP_HIGHBIT_DW_E): Likewise.
+	(FP_FMA_E): Likewise.
+	(_FP_FRAC_HIGH_DW_E): Likewise.
+	* soft-fp/quad.h (_FP_FRACTBITS_DW_Q): New macro.
+	(_FP_WFRACBITS_DW_Q): Likewise.
+	(_FP_WFRACXBITS_DW_Q): Likewise.
+	(_FP_HIGHBIT_DW_Q): Likewise.
+	(FP_FMA_Q): Likewise.
+	(_FP_FRAC_HIGH_DW_Q): Likewise.
+	* soft-fp/fmasf4.c: New file.
+	* soft-fp/fmadf4.c: Likewise.
+	* soft-fp/fmatf4.c: Likewise.
+
 2013-06-28  Liubov Dmitrieva  <liubov.dmitrieva@intel.com>
 
 	* sysdeps/x86_64/multiarch/init-arch.c (__init_cpu_features): Set
diff --git a/ports/ChangeLog.mips b/ports/ChangeLog.mips
index 1f69593a2d..e985de998e 100644
--- a/ports/ChangeLog.mips
+++ b/ports/ChangeLog.mips
@@ -1,3 +1,21 @@
+2013-07-02  Joseph Myers  <joseph@codesourcery.com>
+
+	[BZ #13304]
+	* sysdeps/mips/ieee754/s_fma.c: New file.
+	* sysdeps/mips/ieee754/s_fmaf.c: Likewise.
+	* sysdeps/mips/ieee754/s_fmal.c: Likewise.
+	* sysdeps/mips/mips32/Implies: Add mips/soft-fp.
+	* sysdeps/mips/mips64/n32/s_fma.c: Remove file.
+	* sysdeps/mips/mips64/n64/s_fma.c: Likewise.
+	* sysdeps/mips/mips64/soft-fp/sfp-machine.h (_FP_MUL_MEAT_DW_S):
+	New macro.
+	(_FP_MUL_MEAT_DW_D): Likewise.
+	(_FP_MUL_MEAT_DW_Q): Likewise.
+	* sysdeps/mips/soft-fp/sfp-machine.h (_FP_MUL_MEAT_DW_S): New
+	macro.
+	(_FP_MUL_MEAT_DW_D): Likewise.
+	(_FP_MUL_MEAT_DW_Q): Likewise.
+
 2013-06-28  Ryan S. Arnold  <rsa@linux.vnet.ibm.com>
 
 	* sysdeps/mips/dl-procinfo.h (_dl_procinfo): Add TYPE parameter
diff --git a/ports/sysdeps/mips/ieee754/s_fma.c b/ports/sysdeps/mips/ieee754/s_fma.c
new file mode 100644
index 0000000000..574141416b
--- /dev/null
+++ b/ports/sysdeps/mips/ieee754/s_fma.c
@@ -0,0 +1,5 @@
+#ifdef __mips_hard_float
+# include <sysdeps/ieee754/dbl-64/s_fma.c>
+#else
+# include <soft-fp/fmadf4.c>
+#endif
diff --git a/ports/sysdeps/mips/ieee754/s_fmaf.c b/ports/sysdeps/mips/ieee754/s_fmaf.c
new file mode 100644
index 0000000000..30bcdae620
--- /dev/null
+++ b/ports/sysdeps/mips/ieee754/s_fmaf.c
@@ -0,0 +1,5 @@
+#ifdef __mips_hard_float
+# include <sysdeps/ieee754/dbl-64/s_fmaf.c>
+#else
+# include <soft-fp/fmasf4.c>
+#endif
diff --git a/ports/sysdeps/mips/ieee754/s_fmal.c b/ports/sysdeps/mips/ieee754/s_fmal.c
new file mode 100644
index 0000000000..6b83e914fe
--- /dev/null
+++ b/ports/sysdeps/mips/ieee754/s_fmal.c
@@ -0,0 +1,7 @@
+#include <sgidefs.h>
+
+#if _MIPS_SIM == _ABIO32
+# error "long double fma being compiled for o32 ABI"
+#endif
+
+#include <soft-fp/fmatf4.c>
diff --git a/ports/sysdeps/mips/mips32/Implies b/ports/sysdeps/mips/mips32/Implies
index 6473f2517c..42df98f45e 100644
--- a/ports/sysdeps/mips/mips32/Implies
+++ b/ports/sysdeps/mips/mips32/Implies
@@ -1,3 +1,4 @@
 mips/ieee754
+mips/soft-fp
 mips
 wordsize-32
diff --git a/ports/sysdeps/mips/mips64/n32/s_fma.c b/ports/sysdeps/mips/mips64/n32/s_fma.c
deleted file mode 100644
index 74a1e01fc1..0000000000
--- a/ports/sysdeps/mips/mips64/n32/s_fma.c
+++ /dev/null
@@ -1,6 +0,0 @@
-/* MIPS long double is implemented in software by fp-bit (as of GCC
-   4.7) without support for exceptions or rounding modes, so the fma
-   implementation in terms of long double is slow and will not produce
-   correctly rounding results.  */
-
-#include <sysdeps/ieee754/dbl-64/s_fma.c>
diff --git a/ports/sysdeps/mips/mips64/n64/s_fma.c b/ports/sysdeps/mips/mips64/n64/s_fma.c
deleted file mode 100644
index 74a1e01fc1..0000000000
--- a/ports/sysdeps/mips/mips64/n64/s_fma.c
+++ /dev/null
@@ -1,6 +0,0 @@
-/* MIPS long double is implemented in software by fp-bit (as of GCC
-   4.7) without support for exceptions or rounding modes, so the fma
-   implementation in terms of long double is slow and will not produce
-   correctly rounding results.  */
-
-#include <sysdeps/ieee754/dbl-64/s_fma.c>
diff --git a/ports/sysdeps/mips/mips64/soft-fp/sfp-machine.h b/ports/sysdeps/mips/mips64/soft-fp/sfp-machine.h
index 1bdde5ace8..9cfd6fbb7b 100644
--- a/ports/sysdeps/mips/mips64/soft-fp/sfp-machine.h
+++ b/ports/sysdeps/mips/mips64/soft-fp/sfp-machine.h
@@ -13,6 +13,13 @@
 #define _FP_MUL_MEAT_Q(R,X,Y)					\
   _FP_MUL_MEAT_2_wide_3mul(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm)
 
+#define _FP_MUL_MEAT_DW_S(R,X,Y)				\
+  _FP_MUL_MEAT_DW_1_imm(_FP_WFRACBITS_S,R,X,Y)
+#define _FP_MUL_MEAT_DW_D(R,X,Y)				\
+  _FP_MUL_MEAT_DW_1_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_DW_Q(R,X,Y)				\
+  _FP_MUL_MEAT_DW_2_wide_3mul(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm)
+
 #define _FP_DIV_MEAT_S(R,X,Y)	_FP_DIV_MEAT_1_imm(S,R,X,Y,_FP_DIV_HELP_imm)
 #define _FP_DIV_MEAT_D(R,X,Y)	_FP_DIV_MEAT_1_udiv_norm(D,R,X,Y)
 #define _FP_DIV_MEAT_Q(R,X,Y)	_FP_DIV_MEAT_2_udiv(Q,R,X,Y)
diff --git a/ports/sysdeps/mips/soft-fp/sfp-machine.h b/ports/sysdeps/mips/soft-fp/sfp-machine.h
index 8ccfaa60fd..a60bef7665 100644
--- a/ports/sysdeps/mips/soft-fp/sfp-machine.h
+++ b/ports/sysdeps/mips/soft-fp/sfp-machine.h
@@ -10,6 +10,13 @@
 #define _FP_MUL_MEAT_Q(R,X,Y)				\
   _FP_MUL_MEAT_4_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm)
 
+#define _FP_MUL_MEAT_DW_S(R,X,Y)				\
+  _FP_MUL_MEAT_DW_1_wide(_FP_WFRACBITS_S,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_DW_D(R,X,Y)				\
+  _FP_MUL_MEAT_DW_2_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm)
+#define _FP_MUL_MEAT_DW_Q(R,X,Y)				\
+  _FP_MUL_MEAT_DW_4_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm)
+
 #define _FP_DIV_MEAT_S(R,X,Y)	_FP_DIV_MEAT_1_udiv_norm(S,R,X,Y)
 #define _FP_DIV_MEAT_D(R,X,Y)	_FP_DIV_MEAT_2_udiv(D,R,X,Y)
 #define _FP_DIV_MEAT_Q(R,X,Y)	_FP_DIV_MEAT_4_udiv(Q,R,X,Y)
diff --git a/soft-fp/double.h b/soft-fp/double.h
index 759c2eb661..8653f69138 100644
--- a/soft-fp/double.h
+++ b/soft-fp/double.h
@@ -36,8 +36,10 @@
 
 #if _FP_W_TYPE_SIZE < 64
 #define _FP_FRACTBITS_D		(2 * _FP_W_TYPE_SIZE)
+#define _FP_FRACTBITS_DW_D	(4 * _FP_W_TYPE_SIZE)
 #else
 #define _FP_FRACTBITS_D		_FP_W_TYPE_SIZE
+#define _FP_FRACTBITS_DW_D	(2 * _FP_W_TYPE_SIZE)
 #endif
 
 #define _FP_FRACBITS_D		53
@@ -59,6 +61,11 @@
 #define _FP_OVERFLOW_D		\
 	((_FP_W_TYPE)1 << _FP_WFRACBITS_D % _FP_W_TYPE_SIZE)
 
+#define _FP_WFRACBITS_DW_D	(2 * _FP_WFRACBITS_D)
+#define _FP_WFRACXBITS_DW_D	(_FP_FRACTBITS_DW_D - _FP_WFRACBITS_DW_D)
+#define _FP_HIGHBIT_DW_D	\
+  ((_FP_W_TYPE)1 << (_FP_WFRACBITS_DW_D - 1) % _FP_W_TYPE_SIZE)
+
 typedef float DFtype __attribute__((mode(DF)));
 
 #if _FP_W_TYPE_SIZE < 64
@@ -149,6 +156,7 @@ union _FP_UNION_D
 #define FP_DIV_D(R,X,Y)			_FP_DIV(D,2,R,X,Y)
 #define FP_SQRT_D(R,X)			_FP_SQRT(D,2,R,X)
 #define _FP_SQRT_MEAT_D(R,S,T,X,Q)	_FP_SQRT_MEAT_2(R,S,T,X,Q)
+#define FP_FMA_D(R,X,Y,Z)		_FP_FMA(D,2,4,R,X,Y,Z)
 
 #define FP_CMP_D(r,X,Y,un)	_FP_CMP(D,2,r,X,Y,un)
 #define FP_CMP_EQ_D(r,X,Y)	_FP_CMP_EQ(D,2,r,X,Y)
@@ -160,6 +168,8 @@ union _FP_UNION_D
 #define _FP_FRAC_HIGH_D(X)	_FP_FRAC_HIGH_2(X)
 #define _FP_FRAC_HIGH_RAW_D(X)	_FP_FRAC_HIGH_2(X)
 
+#define _FP_FRAC_HIGH_DW_D(X)	_FP_FRAC_HIGH_4(X)
+
 #else
 
 union _FP_UNION_D
@@ -246,6 +256,7 @@ union _FP_UNION_D
 #define FP_DIV_D(R,X,Y)			_FP_DIV(D,1,R,X,Y)
 #define FP_SQRT_D(R,X)			_FP_SQRT(D,1,R,X)
 #define _FP_SQRT_MEAT_D(R,S,T,X,Q)	_FP_SQRT_MEAT_1(R,S,T,X,Q)
+#define FP_FMA_D(R,X,Y,Z)		_FP_FMA(D,1,2,R,X,Y,Z)
 
 /* The implementation of _FP_MUL_D and _FP_DIV_D should be chosen by
    the target machine.  */
@@ -260,4 +271,6 @@ union _FP_UNION_D
 #define _FP_FRAC_HIGH_D(X)	_FP_FRAC_HIGH_1(X)
 #define _FP_FRAC_HIGH_RAW_D(X)	_FP_FRAC_HIGH_1(X)
 
+#define _FP_FRAC_HIGH_DW_D(X)	_FP_FRAC_HIGH_2(X)
+
 #endif /* W_TYPE_SIZE < 64 */
diff --git a/soft-fp/extended.h b/soft-fp/extended.h
index 74927550eb..c8b1583086 100644
--- a/soft-fp/extended.h
+++ b/soft-fp/extended.h
@@ -33,8 +33,10 @@
 
 #if _FP_W_TYPE_SIZE < 64
 #define _FP_FRACTBITS_E         (4*_FP_W_TYPE_SIZE)
+#define _FP_FRACTBITS_DW_E	(8*_FP_W_TYPE_SIZE)
 #else
 #define _FP_FRACTBITS_E		(2*_FP_W_TYPE_SIZE)
+#define _FP_FRACTBITS_DW_E	(4*_FP_W_TYPE_SIZE)
 #endif
 
 #define _FP_FRACBITS_E		64
@@ -56,6 +58,11 @@
 #define _FP_OVERFLOW_E		\
 	((_FP_W_TYPE)1 << (_FP_WFRACBITS_E % _FP_W_TYPE_SIZE))
 
+#define _FP_WFRACBITS_DW_E	(2 * _FP_WFRACBITS_E)
+#define _FP_WFRACXBITS_DW_E	(_FP_FRACTBITS_DW_E - _FP_WFRACBITS_DW_E)
+#define _FP_HIGHBIT_DW_E	\
+  ((_FP_W_TYPE)1 << (_FP_WFRACBITS_DW_E - 1) % _FP_W_TYPE_SIZE)
+
 typedef float XFtype __attribute__((mode(XF)));
 
 #if _FP_W_TYPE_SIZE < 64
@@ -192,6 +199,7 @@ union _FP_UNION_E
 #define FP_MUL_E(R,X,Y)		_FP_MUL(E,4,R,X,Y)
 #define FP_DIV_E(R,X,Y)		_FP_DIV(E,4,R,X,Y)
 #define FP_SQRT_E(R,X)		_FP_SQRT(E,4,R,X)
+#define FP_FMA_E(R,X,Y,Z)	_FP_FMA(E,4,8,R,X,Y,Z)
 
 /*
  * Square root algorithms:
@@ -258,6 +266,8 @@ union _FP_UNION_E
 #define _FP_FRAC_HIGH_E(X)	(X##_f[2])
 #define _FP_FRAC_HIGH_RAW_E(X)	(X##_f[1])
 
+#define _FP_FRAC_HIGH_DW_E(X)	(X##_f[4])
+
 #else   /* not _FP_W_TYPE_SIZE < 64 */
 union _FP_UNION_E
 {
@@ -383,6 +393,7 @@ union _FP_UNION_E
 #define FP_MUL_E(R,X,Y)		_FP_MUL(E,2,R,X,Y)
 #define FP_DIV_E(R,X,Y)		_FP_DIV(E,2,R,X,Y)
 #define FP_SQRT_E(R,X)		_FP_SQRT(E,2,R,X)
+#define FP_FMA_E(R,X,Y,Z)	_FP_FMA(E,2,4,R,X,Y,Z)
 
 /*
  * Square root algorithms:
@@ -427,4 +438,6 @@ union _FP_UNION_E
 #define _FP_FRAC_HIGH_E(X)	(X##_f1)
 #define _FP_FRAC_HIGH_RAW_E(X)	(X##_f0)
 
+#define _FP_FRAC_HIGH_DW_E(X)	(X##_f[2])
+
 #endif /* not _FP_W_TYPE_SIZE < 64 */
diff --git a/soft-fp/fmadf4.c b/soft-fp/fmadf4.c
new file mode 100644
index 0000000000..ebdc2b1d22
--- /dev/null
+++ b/soft-fp/fmadf4.c
@@ -0,0 +1,56 @@
+/* Implement fma using soft-fp.
+   Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   In addition to the permissions in the GNU Lesser General Public
+   License, the Free Software Foundation gives you unlimited
+   permission to link the compiled version of this file into
+   combinations with other programs, and to distribute those
+   combinations without any restriction coming from the use of this
+   file.  (The Lesser General Public License restrictions do apply in
+   other respects; for example, they cover modification of the file,
+   and distribution when not linked into a combine executable.)
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <math.h>
+#include "soft-fp.h"
+#include "double.h"
+
+double
+__fma (double a, double b, double c)
+{
+  FP_DECL_EX;
+  FP_DECL_D(A); FP_DECL_D(B); FP_DECL_D(C); FP_DECL_D(R);
+  double r;
+
+  FP_INIT_ROUNDMODE;
+  FP_UNPACK_D(A, a);
+  FP_UNPACK_D(B, b);
+  FP_UNPACK_D(C, c);
+  FP_FMA_D(R, A, B, C);
+  FP_PACK_D(r, R);
+  FP_HANDLE_EXCEPTIONS;
+
+  return r;
+}
+#ifndef __fma
+weak_alias (__fma, fma)
+#endif
+
+#ifdef NO_LONG_DOUBLE
+strong_alias (__fma, __fmal)
+weak_alias (__fmal, fmal)
+#endif
diff --git a/soft-fp/fmasf4.c b/soft-fp/fmasf4.c
new file mode 100644
index 0000000000..e8d60fb191
--- /dev/null
+++ b/soft-fp/fmasf4.c
@@ -0,0 +1,51 @@
+/* Implement fmaf using soft-fp.
+   Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   In addition to the permissions in the GNU Lesser General Public
+   License, the Free Software Foundation gives you unlimited
+   permission to link the compiled version of this file into
+   combinations with other programs, and to distribute those
+   combinations without any restriction coming from the use of this
+   file.  (The Lesser General Public License restrictions do apply in
+   other respects; for example, they cover modification of the file,
+   and distribution when not linked into a combine executable.)
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <math.h>
+#include "soft-fp.h"
+#include "single.h"
+
+float
+__fmaf (float a, float b, float c)
+{
+  FP_DECL_EX;
+  FP_DECL_S(A); FP_DECL_S(B); FP_DECL_S(C); FP_DECL_S(R);
+  float r;
+
+  FP_INIT_ROUNDMODE;
+  FP_UNPACK_S(A, a);
+  FP_UNPACK_S(B, b);
+  FP_UNPACK_S(C, c);
+  FP_FMA_S(R, A, B, C);
+  FP_PACK_S(r, R);
+  FP_HANDLE_EXCEPTIONS;
+
+  return r;
+}
+#ifndef __fmaf
+weak_alias (__fmaf, fmaf)
+#endif
diff --git a/soft-fp/fmatf4.c b/soft-fp/fmatf4.c
new file mode 100644
index 0000000000..cf489881d7
--- /dev/null
+++ b/soft-fp/fmatf4.c
@@ -0,0 +1,49 @@
+/* Implement fmal using soft-fp.
+   Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   In addition to the permissions in the GNU Lesser General Public
+   License, the Free Software Foundation gives you unlimited
+   permission to link the compiled version of this file into
+   combinations with other programs, and to distribute those
+   combinations without any restriction coming from the use of this
+   file.  (The Lesser General Public License restrictions do apply in
+   other respects; for example, they cover modification of the file,
+   and distribution when not linked into a combine executable.)
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include <math.h>
+#include "soft-fp.h"
+#include "quad.h"
+
+long double
+__fmal (long double a, long double b, long double c)
+{
+  FP_DECL_EX;
+  FP_DECL_Q(A); FP_DECL_Q(B); FP_DECL_Q(C); FP_DECL_Q(R);
+  long double r;
+
+  FP_INIT_ROUNDMODE;
+  FP_UNPACK_Q(A, a);
+  FP_UNPACK_Q(B, b);
+  FP_UNPACK_Q(C, c);
+  FP_FMA_Q(R, A, B, C);
+  FP_PACK_Q(r, R);
+  FP_HANDLE_EXCEPTIONS;
+
+  return r;
+}
+weak_alias (__fmal, fmal)
diff --git a/soft-fp/op-1.h b/soft-fp/op-1.h
index 8e05e2fab7..a9ad0d62cd 100644
--- a/soft-fp/op-1.h
+++ b/soft-fp/op-1.h
@@ -72,6 +72,7 @@ do {							\
 #define _FP_FRAC_ZEROP_1(X)	(X##_f == 0)
 #define _FP_FRAC_OVERP_1(fs,X)	(X##_f & _FP_OVERFLOW_##fs)
 #define _FP_FRAC_CLEAR_OVERP_1(fs,X)	(X##_f &= ~_FP_OVERFLOW_##fs)
+#define _FP_FRAC_HIGHBIT_DW_1(fs,X)	(X##_f & _FP_HIGHBIT_DW_##fs)
 #define _FP_FRAC_EQ_1(X, Y)	(X##_f == Y##_f)
 #define _FP_FRAC_GE_1(X, Y)	(X##_f >= Y##_f)
 #define _FP_FRAC_GT_1(X, Y)	(X##_f > Y##_f)
@@ -137,9 +138,14 @@ do {							\
 /* Basic.  Assuming the host word size is >= 2*FRACBITS, we can do the
    multiplication immediately.  */
 
-#define _FP_MUL_MEAT_1_imm(wfracbits, R, X, Y)				\
+#define _FP_MUL_MEAT_DW_1_imm(wfracbits, R, X, Y)			\
   do {									\
     R##_f = X##_f * Y##_f;						\
+  } while (0)
+
+#define _FP_MUL_MEAT_1_imm(wfracbits, R, X, Y)				\
+  do {									\
+    _FP_MUL_MEAT_DW_1_imm(wfracbits, R, X, Y);				\
     /* Normalize since we know where the msb of the multiplicands	\
        were (bit B), we know that the msb of the of the product is	\
        at either 2B or 2B-1.  */					\
@@ -148,10 +154,15 @@ do {							\
 
 /* Given a 1W * 1W => 2W primitive, do the extended multiplication.  */
 
+#define _FP_MUL_MEAT_DW_1_wide(wfracbits, R, X, Y, doit)		\
+  do {									\
+    doit(R##_f1, R##_f0, X##_f, Y##_f);					\
+  } while (0)
+
 #define _FP_MUL_MEAT_1_wide(wfracbits, R, X, Y, doit)			\
   do {									\
-    _FP_W_TYPE _Z_f0, _Z_f1;						\
-    doit(_Z_f1, _Z_