<sys/platform/x86.h>: Remove the C preprocessor magic

In <sys/platform/x86.h>, define CPU features as enum instead of using
the C preprocessor magic to make it easier to wrap this functionality
in other languages.  Move the C preprocessor magic to internal header
for better GCC codegen when more than one features are checked in a
single expression as in x86-64 dl-hwcaps-subdirs.c.

1. Rename COMMON_CPUID_INDEX_XXX to CPUID_INDEX_XXX.
2. Move CPUID_INDEX_MAX to sysdeps/x86/include/cpu-features.h.
3. Remove struct cpu_features and __x86_get_cpu_features from
<sys/platform/x86.h>.
4. Add __x86_get_cpuid_feature_leaf to <sys/platform/x86.h> and put it
in libc.
5. Make __get_cpu_features() private to glibc.
6. Replace __x86_get_cpu_features(N) with __get_cpu_features().
7. Add _dl_x86_get_cpu_features to GLIBC_PRIVATE.
8. Use a single enum index for each CPU feature detection.
9. Pass the CPUID feature leaf to __x86_get_cpuid_feature_leaf.
10. Return zero struct cpuid_feature for the older glibc binary with a
smaller CPUID_INDEX_MAX [BZ #27104].
11. Inside glibc, use the C preprocessor magic so that cpu_features data
can be loaded just once leading to more compact code for glibc.

256 bits are used for each CPUID leaf.  Some leaves only contain a few
features.  We can add exceptions to such leaves.  But it will increase
code sizes and it is harder to provide backward/forward compatibilities
when new features are added to such leaves in the future.

When new leaves are added, _rtld_global_ro offsets will change which
leads to race condition during in-place updates. We may avoid in-place
updates by

1. Rename the old glibc.
2. Install the new glibc.
3. Remove the old glibc.

NB: A function, __x86_get_cpuid_feature_leaf , is used to avoid the copy
relocation issue with IFUNC resolver as shown in IFUNC resolver tests.

24 files changed, 1162 insertions, 847 deletions

diff --git a/manual/platform.texi b/manual/platform.texi
index b67683aeb3..6caf68d796 100644
--- a/manual/platform.texi
+++ b/manual/platform.texi
@@ -142,13 +142,10 @@ reserved.
 Facilities specific to X86 that are not specific to a particular
 operating system are declared in @file{sys/platform/x86.h}.
 
-@deftypefun {const struct cpu_features *} __x86_get_cpu_features (unsigned int @var{max})
+@deftypefun {const struct cpuid_feature *} __x86_get_cpuid_feature_leaf (unsigned int @var{leaf})
 @safety{@prelim{}@mtsafe{}@assafe{}@acsafe{}}
 Return a pointer to x86 CPU feature structure used by query macros for x86
-CPU features.  If @var{max} exceeds @code{COMMON_CPUID_INDEX_MAX} which
-is the limit of the CPUID leaves supported by @Theglibc{}, the function
-returns @code{NULL}, indicating that the queried processor feature is
-unsupported by @Theglibc{} run-time.
+CPU feature @var{leaf}.
 @end deftypefun
 
 @deftypefn Macro int HAS_CPU_FEATURE (@var{name})
diff --git a/sysdeps/mach/hurd/i386/ld.abilist b/sysdeps/mach/hurd/i386/ld.abilist
index 751664bcda..7e20c5e7ce 100644
--- a/sysdeps/mach/hurd/i386/ld.abilist
+++ b/sysdeps/mach/hurd/i386/ld.abilist
@@ -16,4 +16,3 @@ GLIBC_2.2.6 _r_debug D 0x14
 GLIBC_2.2.6 abort F
 GLIBC_2.3 ___tls_get_addr F
 GLIBC_2.3 __tls_get_addr F
-GLIBC_2.33 __x86_get_cpu_features F
diff --git a/sysdeps/mach/hurd/i386/libc.abilist b/sysdeps/mach/hurd/i386/libc.abilist
index 7a5eb66b85..509e9b7cb0 100644
--- a/sysdeps/mach/hurd/i386/libc.abilist
+++ b/sysdeps/mach/hurd/i386/libc.abilist
@@ -2191,6 +2191,7 @@ GLIBC_2.32 thrd_current F
 GLIBC_2.32 thrd_equal F
 GLIBC_2.32 thrd_sleep F
 GLIBC_2.32 thrd_yield F
+GLIBC_2.33 __x86_get_cpuid_feature_leaf F
 GLIBC_2.33 fstat F
 GLIBC_2.33 fstat64 F
 GLIBC_2.33 fstatat F
diff --git a/sysdeps/unix/sysv/linux/i386/ld.abilist b/sysdeps/unix/sysv/linux/i386/ld.abilist
index 04655651b0..0478e22071 100644
--- a/sysdeps/unix/sysv/linux/i386/ld.abilist
+++ b/sysdeps/unix/sysv/linux/i386/ld.abilist
@@ -3,4 +3,3 @@ GLIBC_2.1 __libc_stack_end D 0x4
 GLIBC_2.1 _dl_mcount F
 GLIBC_2.3 ___tls_get_addr F
 GLIBC_2.3 __tls_get_addr F
-GLIBC_2.33 __x86_get_cpu_features F
diff --git a/sysdeps/unix/sysv/linux/i386/libc.abilist b/sysdeps/unix/sysv/linux/i386/libc.abilist
index 4f0d3c1eb5..ddc5837059 100644
--- a/sysdeps/unix/sysv/linux/i386/libc.abilist
+++ b/sysdeps/unix/sysv/linux/i386/libc.abilist
@@ -2229,6 +2229,7 @@ GLIBC_2.32 sigabbrev_np F
 GLIBC_2.32 sigdescr_np F
 GLIBC_2.32 strerrordesc_np F
 GLIBC_2.32 strerrorname_np F
+GLIBC_2.33 __x86_get_cpuid_feature_leaf F
 GLIBC_2.33 fstat F
 GLIBC_2.33 fstat64 F
 GLIBC_2.33 fstatat F
diff --git a/sysdeps/unix/sysv/linux/x86_64/64/ld.abilist b/sysdeps/unix/sysv/linux/x86_64/64/ld.abilist
index e632c6198e..d3cdf7611e 100644
--- a/sysdeps/unix/sysv/linux/x86_64/64/ld.abilist
+++ b/sysdeps/unix/sysv/linux/x86_64/64/ld.abilist
@@ -2,4 +2,3 @@ GLIBC_2.2.5 __libc_stack_end D 0x8
 GLIBC_2.2.5 _dl_mcount F
 GLIBC_2.2.5 _r_debug D 0x28
 GLIBC_2.3 __tls_get_addr F
-GLIBC_2.33 __x86_get_cpu_features F
diff --git a/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist b/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist
index 4fff61818b..2744bba4af 100644
--- a/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist
+++ b/sysdeps/unix/sysv/linux/x86_64/64/libc.abilist
@@ -2076,6 +2076,7 @@ GLIBC_2.32 sigabbrev_np F
 GLIBC_2.32 sigdescr_np F
 GLIBC_2.32 strerrordesc_np F
 GLIBC_2.32 strerrorname_np F
+GLIBC_2.33 __x86_get_cpuid_feature_leaf F
 GLIBC_2.33 fstat F
 GLIBC_2.33 fstat64 F
 GLIBC_2.33 fstatat F
diff --git a/sysdeps/unix/sysv/linux/x86_64/tst-glibc-hwcaps-2.c b/sysdeps/unix/sysv/linux/x86_64/tst-glibc-hwcaps-2.c
index fe91bfd224..2d8f7ec462 100644
--- a/sysdeps/unix/sysv/linux/x86_64/tst-glibc-hwcaps-2.c
+++ b/sysdeps/unix/sysv/linux/x86_64/tst-glibc-hwcaps-2.c
@@ -29,8 +29,7 @@ extern int dso_isa_level (void);
 static int
 do_test (void)
 {
-  const struct cpu_features *cpu_features
-    = __x86_get_cpu_features (COMMON_CPUID_INDEX_MAX);
+  const struct cpu_features *cpu_features = __get_cpu_features ();
   unsigned int isa_level = get_isa_level (cpu_features);
   bool has_isa_baseline = ((isa_level & GNU_PROPERTY_X86_ISA_1_BASELINE)
 			   == GNU_PROPERTY_X86_ISA_1_BASELINE);
diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/ld.abilist b/sysdeps/unix/sysv/linux/x86_64/x32/ld.abilist
index 9d9d0befde..c70bccf782 100644
--- a/sysdeps/unix/sysv/linux/x86_64/x32/ld.abilist
+++ b/sysdeps/unix/sysv/linux/x86_64/x32/ld.abilist
@@ -2,4 +2,3 @@ GLIBC_2.16 __libc_stack_end D 0x4
 GLIBC_2.16 __tls_get_addr F
 GLIBC_2.16 _dl_mcount F
 GLIBC_2.16 _r_debug D 0x14
-GLIBC_2.33 __x86_get_cpu_features F
diff --git a/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist b/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist
index 102ed47a9c..ce2f4fb72b 100644
--- a/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist
+++ b/sysdeps/unix/sysv/linux/x86_64/x32/libc.abilist
@@ -2173,6 +2173,7 @@ GLIBC_2.32 sigabbrev_np F
 GLIBC_2.32 sigdescr_np F
 GLIBC_2.32 strerrordesc_np F
 GLIBC_2.32 strerrorname_np F
+GLIBC_2.33 __x86_get_cpuid_feature_leaf F
 GLIBC_2.33 fstat F
 GLIBC_2.33 fstat64 F
 GLIBC_2.33 fstatat F
diff --git a/sysdeps/x86/Makefile b/sysdeps/x86/Makefile
index adaa2a92cd..86e980debc 100644
--- a/sysdeps/x86/Makefile
+++ b/sysdeps/x86/Makefile
@@ -3,6 +3,7 @@ gen-as-const-headers += cpu-features-offsets.sym
 endif
 
 ifeq ($(subdir),elf)
+sysdep_routines += get-cpuid-feature-leaf
 sysdep-dl-routines += dl-get-cpu-features
 sysdep_headers += sys/platform/x86.h
 
diff --git a/sysdeps/x86/Versions b/sysdeps/x86/Versions
index 59db578a9d..4b10c4b5d7 100644
--- a/sysdeps/x86/Versions
+++ b/sysdeps/x86/Versions
@@ -1,5 +1,10 @@
 ld {
+  GLIBC_PRIVATE {
+    _dl_x86_get_cpu_features;
+  }
+}
+libc {
   GLIBC_2.33 {
-    __x86_get_cpu_features;
+    __x86_get_cpuid_feature_leaf;
   }
 }
diff --git a/sysdeps/x86/bits/platform/x86.h b/sysdeps/x86/bits/platform/x86.h
new file mode 100644
index 0000000000..8f423ae721
--- /dev/null
+++ b/sysdeps/x86/bits/platform/x86.h
@@ -0,0 +1,299 @@
+/* Constants and data structures for x86 CPU features.
+   This file is part of the GNU C Library.
+   Copyright (C) 2008-2020 Free Software Foundation, Inc.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#ifndef _SYS_PLATFORM_X86_H
+# error "Never include <bits/platform/x86.h> directly; use <sys/platform/x86.h> instead."
+#endif
+
+enum
+{
+  CPUID_INDEX_1 = 0,
+  CPUID_INDEX_7,
+  CPUID_INDEX_80000001,
+  CPUID_INDEX_D_ECX_1,
+  CPUID_INDEX_80000007,
+  CPUID_INDEX_80000008,
+  CPUID_INDEX_7_ECX_1,
+  CPUID_INDEX_19
+};
+
+struct cpuid_feature
+{
+  unsigned int cpuid_array[4];
+  unsigned int usable_array[4];
+};
+
+enum cpuid_register_index
+{
+  cpuid_register_index_eax = 0,
+  cpuid_register_index_ebx,
+  cpuid_register_index_ecx,
+  cpuid_register_index_edx
+};
+
+/* CPU features.  */
+
+enum
+{
+  x86_cpu_index_1_ecx
+    = (CPUID_INDEX_1 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_ecx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_SSE3			= x86_cpu_index_1_ecx,
+  x86_cpu_PCLMULQDQ		= x86_cpu_index_1_ecx + 1,
+  x86_cpu_DTES64		= x86_cpu_index_1_ecx + 2,
+  x86_cpu_MONITOR		= x86_cpu_index_1_ecx + 3,
+  x86_cpu_DS_CPL		= x86_cpu_index_1_ecx + 4,
+  x86_cpu_VMX			= x86_cpu_index_1_ecx + 5,
+  x86_cpu_SMX			= x86_cpu_index_1_ecx + 6,
+  x86_cpu_EIST			= x86_cpu_index_1_ecx + 7,
+  x86_cpu_TM2			= x86_cpu_index_1_ecx + 8,
+  x86_cpu_SSSE3			= x86_cpu_index_1_ecx + 9,
+  x86_cpu_CNXT_ID		= x86_cpu_index_1_ecx + 10,
+  x86_cpu_SDBG			= x86_cpu_index_1_ecx + 11,
+  x86_cpu_FMA			= x86_cpu_index_1_ecx + 12,
+  x86_cpu_CMPXCHG16B		= x86_cpu_index_1_ecx + 13,
+  x86_cpu_XTPRUPDCTRL		= x86_cpu_index_1_ecx + 14,
+  x86_cpu_PDCM			= x86_cpu_index_1_ecx + 15,
+  x86_cpu_INDEX_1_ECX_16	= x86_cpu_index_1_ecx + 16,
+  x86_cpu_PCID			= x86_cpu_index_1_ecx + 17,
+  x86_cpu_DCA			= x86_cpu_index_1_ecx + 18,
+  x86_cpu_SSE4_1		= x86_cpu_index_1_ecx + 19,
+  x86_cpu_SSE4_2		= x86_cpu_index_1_ecx + 20,
+  x86_cpu_X2APIC		= x86_cpu_index_1_ecx + 21,
+  x86_cpu_MOVBE			= x86_cpu_index_1_ecx + 22,
+  x86_cpu_POPCNT		= x86_cpu_index_1_ecx + 23,
+  x86_cpu_TSC_DEADLINE		= x86_cpu_index_1_ecx + 24,
+  x86_cpu_AES			= x86_cpu_index_1_ecx + 25,
+  x86_cpu_XSAVE			= x86_cpu_index_1_ecx + 26,
+  x86_cpu_OSXSAVE		= x86_cpu_index_1_ecx + 27,
+  x86_cpu_AVX			= x86_cpu_index_1_ecx + 28,
+  x86_cpu_F16C			= x86_cpu_index_1_ecx + 29,
+  x86_cpu_RDRAND		= x86_cpu_index_1_ecx + 30,
+  x86_cpu_INDEX_1_ECX_31	= x86_cpu_index_1_ecx + 31,
+
+  x86_cpu_index_1_edx
+    = (CPUID_INDEX_1 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_edx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_FPU			= x86_cpu_index_1_edx,
+  x86_cpu_VME			= x86_cpu_index_1_edx + 1,
+  x86_cpu_DE			= x86_cpu_index_1_edx + 2,
+  x86_cpu_PSE			= x86_cpu_index_1_edx + 3,
+  x86_cpu_TSC			= x86_cpu_index_1_edx + 4,
+  x86_cpu_MSR			= x86_cpu_index_1_edx + 5,
+  x86_cpu_PAE			= x86_cpu_index_1_edx + 6,
+  x86_cpu_MCE			= x86_cpu_index_1_edx + 7,
+  x86_cpu_CX8			= x86_cpu_index_1_edx + 8,
+  x86_cpu_APIC			= x86_cpu_index_1_edx + 9,
+  x86_cpu_INDEX_1_EDX_10	= x86_cpu_index_1_edx + 10,
+  x86_cpu_SEP			= x86_cpu_index_1_edx + 11,
+  x86_cpu_MTRR			= x86_cpu_index_1_edx + 12,
+  x86_cpu_PGE			= x86_cpu_index_1_edx + 13,
+  x86_cpu_MCA			= x86_cpu_index_1_edx + 14,
+  x86_cpu_CMOV			= x86_cpu_index_1_edx + 15,
+  x86_cpu_PAT			= x86_cpu_index_1_edx + 16,
+  x86_cpu_PSE_36		= x86_cpu_index_1_edx + 17,
+  x86_cpu_PSN			= x86_cpu_index_1_edx + 18,
+  x86_cpu_CLFSH			= x86_cpu_index_1_edx + 19,
+  x86_cpu_INDEX_1_EDX_20	= x86_cpu_index_1_edx + 20,
+  x86_cpu_DS			= x86_cpu_index_1_edx + 21,
+  x86_cpu_ACPI			= x86_cpu_index_1_edx + 22,
+  x86_cpu_MMX			= x86_cpu_index_1_edx + 23,
+  x86_cpu_FXSR			= x86_cpu_index_1_edx + 24,
+  x86_cpu_SSE			= x86_cpu_index_1_edx + 25,
+  x86_cpu_SSE2			= x86_cpu_index_1_edx + 26,
+  x86_cpu_SS			= x86_cpu_index_1_edx + 27,
+  x86_cpu_HTT			= x86_cpu_index_1_edx + 28,
+  x86_cpu_TM			= x86_cpu_index_1_edx + 29,
+  x86_cpu_INDEX_1_EDX_30	= x86_cpu_index_1_edx + 30,
+  x86_cpu_PBE			= x86_cpu_index_1_edx + 31,
+
+  x86_cpu_index_7_ebx
+    = (CPUID_INDEX_7 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_ebx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_FSGSBASE		= x86_cpu_index_7_ebx,
+  x86_cpu_TSC_ADJUST		= x86_cpu_index_7_ebx + 1,
+  x86_cpu_SGX			= x86_cpu_index_7_ebx + 2,
+  x86_cpu_BMI1			= x86_cpu_index_7_ebx + 3,
+  x86_cpu_HLE			= x86_cpu_index_7_ebx + 4,
+  x86_cpu_AVX2			= x86_cpu_index_7_ebx + 5,
+  x86_cpu_INDEX_7_EBX_6		= x86_cpu_index_7_ebx + 6,
+  x86_cpu_SMEP			= x86_cpu_index_7_ebx + 7,
+  x86_cpu_BMI2			= x86_cpu_index_7_ebx + 8,
+  x86_cpu_ERMS			= x86_cpu_index_7_ebx + 9,
+  x86_cpu_INVPCID		= x86_cpu_index_7_ebx + 10,
+  x86_cpu_RTM			= x86_cpu_index_7_ebx + 11,
+  x86_cpu_RDT_M			= x86_cpu_index_7_ebx + 12,
+  x86_cpu_DEPR_FPU_CS_DS	= x86_cpu_index_7_ebx + 13,
+  x86_cpu_MPX			= x86_cpu_index_7_ebx + 14,
+  x86_cpu_RDT_A			= x86_cpu_index_7_ebx + 15,
+  x86_cpu_AVX512F		= x86_cpu_index_7_ebx + 16,
+  x86_cpu_AVX512DQ		= x86_cpu_index_7_ebx + 17,
+  x86_cpu_RDSEED		= x86_cpu_index_7_ebx + 18,
+  x86_cpu_ADX			= x86_cpu_index_7_ebx + 19,
+  x86_cpu_SMAP			= x86_cpu_index_7_ebx + 20,
+  x86_cpu_AVX512_IFMA		= x86_cpu_index_7_ebx + 21,
+  x86_cpu_INDEX_7_EBX_22	= x86_cpu_index_7_ebx + 22,
+  x86_cpu_CLFLUSHOPT		= x86_cpu_index_7_ebx + 23,
+  x86_cpu_CLWB			= x86_cpu_index_7_ebx + 24,
+  x86_cpu_TRACE			= x86_cpu_index_7_ebx + 25,
+  x86_cpu_AVX512PF		= x86_cpu_index_7_ebx + 26,
+  x86_cpu_AVX512ER		= x86_cpu_index_7_ebx + 27,
+  x86_cpu_AVX512CD		= x86_cpu_index_7_ebx + 28,
+  x86_cpu_SHA			= x86_cpu_index_7_ebx + 29,
+  x86_cpu_AVX512BW		= x86_cpu_index_7_ebx + 30,
+  x86_cpu_AVX512VL		= x86_cpu_index_7_ebx + 31,
+
+  x86_cpu_index_7_ecx
+    = (CPUID_INDEX_7 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_ecx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_PREFETCHWT1		= x86_cpu_index_7_ecx,
+  x86_cpu_AVX512_VBMI		= x86_cpu_index_7_ecx + 1,
+  x86_cpu_UMIP			= x86_cpu_index_7_ecx + 2,
+  x86_cpu_PKU			= x86_cpu_index_7_ecx + 3,
+  x86_cpu_OSPKE			= x86_cpu_index_7_ecx + 4,
+  x86_cpu_WAITPKG		= x86_cpu_index_7_ecx + 5,
+  x86_cpu_AVX512_VBMI2		= x86_cpu_index_7_ecx + 6,
+  x86_cpu_SHSTK			= x86_cpu_index_7_ecx + 7,
+  x86_cpu_GFNI			= x86_cpu_index_7_ecx + 8,
+  x86_cpu_VAES			= x86_cpu_index_7_ecx + 9,
+  x86_cpu_VPCLMULQDQ		= x86_cpu_index_7_ecx + 10,
+  x86_cpu_AVX512_VNNI		= x86_cpu_index_7_ecx + 11,
+  x86_cpu_AVX512_BITALG		= x86_cpu_index_7_ecx + 12,
+  x86_cpu_INDEX_7_ECX_13	= x86_cpu_index_7_ecx + 13,
+  x86_cpu_AVX512_VPOPCNTDQ	= x86_cpu_index_7_ecx + 14,
+  x86_cpu_INDEX_7_ECX_1		= x86_cpu_index_7_ecx + 15,
+  x86_cpu_INDEX_7_ECX_16	= x86_cpu_index_7_ecx + 16,
+/* Note: Bits 17-21: The value of MAWAU used by the BNDLDX and BNDSTX
+   instructions in 64-bit mode.  */
+  x86_cpu_RDPID			= x86_cpu_index_7_ecx + 22,
+  x86_cpu_KL			= x86_cpu_index_7_ecx + 23,
+  x86_cpu_INDEX_7_ECX_24	= x86_cpu_index_7_ecx + 24,
+  x86_cpu_CLDEMOTE		= x86_cpu_index_7_ecx + 25,
+  x86_cpu_INDEX_7_ECX_26	= x86_cpu_index_7_ecx + 26,
+  x86_cpu_MOVDIRI		= x86_cpu_index_7_ecx + 27,
+  x86_cpu_MOVDIR64B		= x86_cpu_index_7_ecx + 28,
+  x86_cpu_ENQCMD		= x86_cpu_index_7_ecx + 29,
+  x86_cpu_SGX_LC		= x86_cpu_index_7_ecx + 30,
+  x86_cpu_PKS			= x86_cpu_index_7_ecx + 31,
+
+  x86_cpu_index_7_edx
+    = (CPUID_INDEX_7 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_edx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_INDEX_7_EDX_0		= x86_cpu_index_7_edx,
+  x86_cpu_INDEX_7_EDX_1		= x86_cpu_index_7_edx + 1,
+  x86_cpu_AVX512_4VNNIW		= x86_cpu_index_7_edx + 2,
+  x86_cpu_AVX512_4FMAPS		= x86_cpu_index_7_edx + 3,
+  x86_cpu_FSRM			= x86_cpu_index_7_edx + 4,
+  x86_cpu_UINTR			= x86_cpu_index_7_edx + 5,
+  x86_cpu_INDEX_7_EDX_6		= x86_cpu_index_7_edx + 6,
+  x86_cpu_INDEX_7_EDX_7		= x86_cpu_index_7_edx + 7,
+  x86_cpu_AVX512_VP2INTERSECT	= x86_cpu_index_7_edx + 8,
+  x86_cpu_INDEX_7_EDX_9		= x86_cpu_index_7_edx + 9,
+  x86_cpu_MD_CLEAR		= x86_cpu_index_7_edx + 10,
+  x86_cpu_INDEX_7_EDX_11	= x86_cpu_index_7_edx + 11,
+  x86_cpu_INDEX_7_EDX_12	= x86_cpu_index_7_edx + 12,
+  x86_cpu_INDEX_7_EDX_13	= x86_cpu_index_7_edx + 13,
+  x86_cpu_SERIALIZE		= x86_cpu_index_7_edx + 14,
+  x86_cpu_HYBRID		= x86_cpu_index_7_edx + 15,
+  x86_cpu_TSXLDTRK		= x86_cpu_index_7_edx + 16,
+  x86_cpu_INDEX_7_EDX_17	= x86_cpu_index_7_edx + 17,
+  x86_cpu_PCONFIG		= x86_cpu_index_7_edx + 18,
+  x86_cpu_INDEX_7_EDX_19	= x86_cpu_index_7_edx + 19,
+  x86_cpu_IBT			= x86_cpu_index_7_edx + 20,
+  x86_cpu_INDEX_7_EDX_21	= x86_cpu_index_7_edx + 21,
+  x86_cpu_AMX_BF16		= x86_cpu_index_7_edx + 22,
+  x86_cpu_AVX512_FP16		= x86_cpu_index_7_edx + 23,
+  x86_cpu_AMX_TILE		= x86_cpu_index_7_edx + 24,
+  x86_cpu_AMX_INT8		= x86_cpu_index_7_edx + 25,
+  x86_cpu_IBRS_IBPB		= x86_cpu_index_7_edx + 26,
+  x86_cpu_STIBP			= x86_cpu_index_7_edx + 27,
+  x86_cpu_L1D_FLUSH		= x86_cpu_index_7_edx + 28,
+  x86_cpu_ARCH_CAPABILITIES	= x86_cpu_index_7_edx + 29,
+  x86_cpu_CORE_CAPABILITIES	= x86_cpu_index_7_edx + 30,
+  x86_cpu_SSBD			= x86_cpu_index_7_edx + 31,
+
+  x86_cpu_index_80000001_ecx
+    = (CPUID_INDEX_80000001 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_ecx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_LAHF64_SAHF64		= x86_cpu_index_80000001_ecx,
+  x86_cpu_SVM			= x86_cpu_index_80000001_ecx + 2,
+  x86_cpu_LZCNT			= x86_cpu_index_80000001_ecx + 5,
+  x86_cpu_SSE4A			= x86_cpu_index_80000001_ecx + 6,
+  x86_cpu_PREFETCHW		= x86_cpu_index_80000001_ecx + 8,
+  x86_cpu_XOP			= x86_cpu_index_80000001_ecx + 11,
+  x86_cpu_LWP			= x86_cpu_index_80000001_ecx + 15,
+  x86_cpu_FMA4			= x86_cpu_index_80000001_ecx + 16,
+  x86_cpu_TBM			= x86_cpu_index_80000001_ecx + 20,
+
+  x86_cpu_index_80000001_edx
+    = (CPUID_INDEX_80000001 * 8 * 4 * sizeof (unsigned int)
+       + cpuid_register_index_edx * 8 * sizeof (unsigned int)),
+
+  x86_cpu_SYSCALL_SYSRET	= x86_cpu_index_80000001_edx + 11,
+  x86_cpu_NX			= x86_cpu_index_80000001_edx + 20,
+  x86_cpu_PAGE1GB		= x86_cpu_index_80000001_edx + 26,
+  x86_cpu_RDTSCP		=