diff options
25 files changed, 643 insertions, 240 deletions
@@ -1,3 +1,41 @@ +2017-06-21 H.J. Lu <hongjiu.lu@intel.com> + + * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add + strcmp-sse2, strcmp-sse4_2, strncmp-sse2, strncmp-sse4_2, + strcasecmp_l-sse2, strcasecmp_l-sse4_2, strcasecmp_l-avx, + strncase_l-sse2, strncase_l-sse4_2 and strncase_l-avx. + * sysdeps/x86_64/multiarch/ifunc-strcasecmp.h: New file. + * sysdeps/x86_64/multiarch/strcasecmp.c: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l-avx.S: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l.c: Likewise. + * sysdeps/x86_64/multiarch/strcmp-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strcmp-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strcmp.c: Likewise. + * sysdeps/x86_64/multiarch/strncase.c: Likewise. + * sysdeps/x86_64/multiarch/strncase_l-avx.S : Likewise. + * sysdeps/x86_64/multiarch/strncase_l-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strncase_l-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strncase_l.c: Likewise. + * sysdeps/x86_64/multiarch/strncmp-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/strncmp-sse4_2.S: Likewise. + * sysdeps/x86_64/multiarch/strncmp.c: Likewise. + * sysdeps/x86_64/multiarch/strcasecmp_l.S: Removed. + * sysdeps/x86_64/multiarch/strcmp.S: Likewise. + * sysdeps/x86_64/multiarch/strncase_l.S: Likewise. + * sysdeps/x86_64/multiarch/strncmp.S: Likewise. + * sysdeps/x86_64/multiarch/strcmp-sse42.S: Include <sysdep.h>. + (STRCMP_SSE42): New. Defined to __strcmp_sse42 if not defined. + [USE_AS_STRCASECMP_L || USE_AS_STRNCASECMP_L]: Include + "locale-defines.h". + (UPDATE_STRNCMP_COUNTER): New. + (SECTION): Likewise. + (GLABEL): Likewise. + (LABEL): Likewise. + * sysdeps/x86_64/multiarch/strncmp-ssse3.S: Rewrite and enable + for libc.a. + 2017-06-21 Joseph Myers <joseph@codesourcery.com> [BZ #21622] diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 2c54c5cc23..68257c4017 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -4,8 +4,10 @@ endif ifeq ($(subdir),string) -sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ - strcmp-sse2-unaligned strncmp-ssse3 \ +sysdep_routines += strncat-c stpncpy-c strncpy-c \ + strcmp-sse2 strcmp-sse2-unaligned strcmp-ssse3 \ + strcmp-sse4_2 \ + strncmp-sse2 strncmp-ssse3 strncmp-sse4_2 \ memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \ memrchr-sse2 memrchr-avx2 \ memcmp-sse2 \ @@ -14,11 +16,15 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ memmove-ssse3 \ memcpy-ssse3-back \ memmove-ssse3-back \ - memmove-avx512-no-vzeroupper strcasecmp_l-ssse3 \ + memmove-avx512-no-vzeroupper \ + strcasecmp_l-sse2 strcasecmp_l-ssse3 \ + strcasecmp_l-sse4_2 strcasecmp_l-avx \ + strncase_l-sse2 strncase_l-ssse3 \ + strncase_l-sse4_2 strncase_l-avx \ strchr-sse2 strchrnul-sse2 strchr-avx2 strchrnul-avx2 \ strrchr-sse2 strrchr-avx2 \ strlen-sse2 strnlen-sse2 strlen-avx2 strnlen-avx2 \ - strncase_l-ssse3 strcat-ssse3 strncat-ssse3\ + strcat-ssse3 strncat-ssse3\ strcpy-sse2 stpcpy-sse2 \ strcpy-ssse3 strncpy-ssse3 stpcpy-ssse3 stpncpy-ssse3 \ strcpy-sse2-unaligned strncpy-sse2-unaligned \ diff --git a/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h new file mode 100644 index 0000000000..adecd6d8e0 --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-strcasecmp.h @@ -0,0 +1,43 @@ +/* Common definition for strcasecmp famly ifunc selections. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse42) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURES_ARCH_P (cpu_features, AVX_Usable)) + return OPTIMIZE (avx); + + if (CPU_FEATURES_CPU_P (cpu_features, SSE4_2) + && !CPU_FEATURES_ARCH_P (cpu_features, Slow_SSE4_2)) + return OPTIMIZE (sse42); + + if (CPU_FEATURES_CPU_P (cpu_features, SSSE3)) + return OPTIMIZE (ssse3); + + return OPTIMIZE (sse2); +} diff --git a/sysdeps/x86_64/multiarch/strcasecmp.c b/sysdeps/x86_64/multiarch/strcasecmp.c new file mode 100644 index 0000000000..8170bdee69 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp.c @@ -0,0 +1,39 @@ +/* Multiple versions of strcasecmp. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strcasecmp __redirect_strcasecmp +# define __strcasecmp __redirect___strcasecmp +# include <string.h> +# undef strcasecmp +# undef __strcasecmp + +# define SYMBOL_NAME strcasecmp +# include "ifunc-strcasecmp.h" + +libc_ifunc_redirected (__redirect_strcasecmp, __strcasecmp, + IFUNC_SELECTOR ()); + +weak_alias (__strcasecmp, strcasecmp) +# ifdef SHARED +__hidden_ver1 (__strcasecmp, __GI___strcasecmp, __redirect___strcasecmp) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-avx.S b/sysdeps/x86_64/multiarch/strcasecmp_l-avx.S new file mode 100644 index 0000000000..60f5b50c66 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-avx.S @@ -0,0 +1,22 @@ +/* strcasecmp_l optimized with AVX. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strcasecmp_l_avx +#define USE_AVX 1 +#define USE_AS_STRCASECMP_L +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S new file mode 100644 index 0000000000..0b04184c5c --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-sse2.S @@ -0,0 +1,23 @@ +/* strcasecmp_l optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP __strcasecmp_l_sse2 +#define USE_AS_STRCASECMP_L +#define NO_NOLOCALE_ALIAS +#define __strcasecmp __strcasecmp_sse2 +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S b/sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S new file mode 100644 index 0000000000..92a660be01 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l-sse4_2.S @@ -0,0 +1,21 @@ +/* strcasecmp_l optimized with SSE4.2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#define STRCMP_SSE42 __strcasecmp_l_sse42 +#define USE_AS_STRCASECMP_L +#include "strcmp-sse42.S" diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l.S b/sysdeps/x86_64/multiarch/strcasecmp_l.S deleted file mode 100644 index 49f5b9fd95..0000000000 --- a/sysdeps/x86_64/multiarch/strcasecmp_l.S +++ /dev/null @@ -1,8 +0,0 @@ -/* Multiple versions of strcasecmp and strcasecmp_l - All versions must be listed in ifunc-impl-list.c. */ -#define STRCMP __strcasecmp_l -#define USE_AS_STRCASECMP_L -#include "strcmp.S" - -weak_alias (__strcasecmp_l, strcasecmp_l) -libc_hidden_def (strcasecmp_l) diff --git a/sysdeps/x86_64/multiarch/strcasecmp_l.c b/sysdeps/x86_64/multiarch/strcasecmp_l.c new file mode 100644 index 0000000000..f28db6cd20 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcasecmp_l.c @@ -0,0 +1,40 @@ +/* Multiple versions of strcasecmp_l. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define strcasecmp_l __redirect_strcasecmp_l +# define __strcasecmp_l __redirect___strcasecmp_l +# include <string.h> +# undef strcasecmp_l +# undef __strcasecmp_l + +# define SYMBOL_NAME strcasecmp_l +# include "ifunc-strcasecmp.h" + +libc_ifunc_redirected (__redirect_strcasecmp_l, __strcasecmp_l, + IFUNC_SELECTOR ()); + +weak_alias (__strcasecmp_l, strcasecmp_l) +# ifdef SHARED +__hidden_ver1 (__strcasecmp_l, __GI___strcasecmp_l, + __redirect___strcasecmp_l) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/strcmp-sse2.S b/sysdeps/x86_64/multiarch/strcmp-sse2.S new file mode 100644 index 0000000000..834ba3f093 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcmp-sse2.S @@ -0,0 +1,28 @@ +/* strcmp optimized with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# include <sysdep.h> + +# define STRCMP __strcmp_sse2 + +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(strcmp) +#endif + +#include <sysdeps/x86_64/strcmp.S> diff --git a/sysdeps/x86_64/multiarch/strcmp-sse42.S b/sysdeps/x86_64/multiarch/strcmp-sse42.S index ed26d4a8fb..4aeb14e175 100644 --- a/sysdeps/x86_64/multiarch/strcmp-sse42.S +++ b/sysdeps/x86_64/multiarch/strcmp-sse42.S @@ -17,6 +17,40 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ +#include <sysdep.h> + +#ifndef STRCMP_SSE42 +# define STRCMP_SSE42 __strcmp_sse42 +#endif + +#if defined USE_AS_STRCASECMP_L || defined USE_AS_STRNCASECMP_L +# include "locale-defines.h" +#endif + +#if defined USE_AS_STRNCMP || defined USE_AS_STRNCASECMP_L +/* Since the counter, %r11, is unsigned, we branch to strcmp_exitz + if the new counter > the old one or is 0. */ +# define UPDATE_STRNCMP_COUNTER \ + /* calculate left number to compare */ \ + lea -16(%rcx, %r11), %r9; \ + cmp %r9, %r11; \ + jb LABEL(strcmp_exitz); \ + test %r9, %r9; \ + je LABEL(strcmp_exitz); \ + mov %r9, %r11 +#else +# define UPDATE_STRNCMP_COUNTER +#endif + +#ifdef USE_AVX +# define SECTION avx +# define GLABEL(l) l##_avx +#else +# define SECTION sse4.2 +# define GLABEL(l) l##_sse42 +#endif + +#define LABEL(l) .L##l /* We use 0x1a: _SIDD_SBYTE_OPS diff --git a/sysdeps/x86_64/multiarch/strcmp-sse4_2.S b/sysdeps/x86_64/multiarch/strcmp-sse4_2.S new file mode 100644 index 0000000000..acf0079a78 --- /dev/null +++ b/sysdeps/x86_64/multiarch/strcmp-sse4_2.S @@ -0,0 +1,21 @@ +/* strcmp optimized with SSE4.2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# include "strcmp-sse42.S" +#endif diff --git a/sysdeps/x86_64/multiarch/strcmp.S b/sysdeps/x86_64/multiarch/strcmp.S deleted file mode 100644 index 54f8f7dd44..0000000000 --- a/sysdeps/x86_64/multiarch/strcmp.S +++ /dev/null @@ -1,209 +0,0 @@ -/* Multiple versions of strcmp - Copyright (C) 2009-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -#ifdef USE_AS_STRNCMP -/* Since the counter, %r11, is unsigned, we branch to strcmp_exitz - if the new counter > the old one or is 0. */ -# define UPDATE_STRNCMP_COUNTER \ - /* calculate left number to compare */ \ - lea -16(%rcx, %r11), %r9; \ - cmp %r9, %r11; \ - jb LABEL(strcmp_exitz); \ - test %r9, %r9; \ - je LABEL(strcmp_exitz); \ - mov %r9, %r11 - -# define STRCMP_SSE42 __strncmp_sse42 -# define STRCMP_SSSE3 __strncmp_ssse3 -# define STRCMP_SSE2 __strncmp_sse2 -# define __GI_STRCMP __GI_strncmp -#elif defined USE_AS_STRCASECMP_L |