diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/build-many-glibcs.py | 1142 |
1 files changed, 1142 insertions, 0 deletions
diff --git a/scripts/build-many-glibcs.py b/scripts/build-many-glibcs.py new file mode 100755 index 0000000000..4b4e15ca0b --- /dev/null +++ b/scripts/build-many-glibcs.py @@ -0,0 +1,1142 @@ +#!/usr/bin/python3 +# Build many configurations of glibc. +# Copyright (C) 2016 Free Software Foundation, Inc. +# This file is part of the GNU C Library. +# +# The GNU C Library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# The GNU C Library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with the GNU C Library; if not, see +# <http://www.gnu.org/licenses/>. + +"""Build many configurations of glibc. + +This script takes as arguments a directory name (containing a src +subdirectory with sources of the relevant toolchain components) and a +description of what to do: 'checkout', to check out sources into that +directory, 'host-libraries', to build libraries required by the +toolchain, 'compilers', to build cross-compilers for various +configurations, or 'glibcs', to build glibc for various configurations +and run the compilation parts of the testsuite. Subsequent arguments +name the versions of components to check out (<component>-<version), +for 'checkout', or, for actions other than 'checkout', name +configurations for which compilers or glibc are to be built. +""" + +import argparse +import os +import os.path +import re +import shutil +import stat +import subprocess +import sys +import urllib.request + + +class Context: + + """The global state associated with builds in a given directory.""" + + def __init__(self, topdir, parallelism, keep, action): + """Initialize the context.""" + self.topdir = topdir + self.parallelism = parallelism + self.keep = keep + self.srcdir = os.path.join(topdir, 'src') + self.installdir = os.path.join(topdir, 'install') + self.host_libraries_installdir = os.path.join(self.installdir, + 'host-libraries') + self.builddir = os.path.join(topdir, 'build') + self.logsdir = os.path.join(topdir, 'logs') + self.makefile = os.path.join(self.builddir, 'Makefile') + self.wrapper = os.path.join(self.builddir, 'wrapper') + self.save_logs = os.path.join(self.builddir, 'save-logs') + if action != 'checkout': + self.build_triplet = self.get_build_triplet() + self.glibc_version = self.get_glibc_version() + self.configs = {} + self.glibc_configs = {} + self.makefile_pieces = ['.PHONY: all\n'] + self.add_all_configs() + + def get_build_triplet(self): + """Determine the build triplet with config.guess.""" + config_guess = os.path.join(self.component_srcdir('gcc'), + 'config.guess') + cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE, + check=True, universal_newlines=True).stdout + return cg_out.rstrip() + + def get_glibc_version(self): + """Determine the glibc version number (major.minor).""" + version_h = os.path.join(self.component_srcdir('glibc'), 'version.h') + with open(version_h, 'r') as f: + lines = f.readlines() + starttext = '#define VERSION "' + for l in lines: + if l.startswith(starttext): + l = l[len(starttext):] + l = l.rstrip('"\n') + m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l) + return '%s.%s' % m.group(1, 2) + print('error: could not determine glibc version') + exit(1) + + def add_all_configs(self): + """Add all known glibc build configurations.""" + self.add_config(arch='aarch64', + os_name='linux-gnu') + self.add_config(arch='aarch64_be', + os_name='linux-gnu') + self.add_config(arch='alpha', + os_name='linux-gnu') + self.add_config(arch='arm', + os_name='linux-gnueabi') + self.add_config(arch='armeb', + os_name='linux-gnueabi') + self.add_config(arch='armeb', + os_name='linux-gnueabi', + variant='be8', + gcc_cfg=['--with-arch=armv7-a']) + self.add_config(arch='arm', + os_name='linux-gnueabihf') + self.add_config(arch='armeb', + os_name='linux-gnueabihf') + self.add_config(arch='armeb', + os_name='linux-gnueabihf', + variant='be8', + gcc_cfg=['--with-arch=armv7-a']) + self.add_config(arch='hppa', + os_name='linux-gnu') + self.add_config(arch='ia64', + os_name='linux-gnu', + first_gcc_cfg=['--with-system-libunwind']) + self.add_config(arch='m68k', + os_name='linux-gnu', + gcc_cfg=['--disable-multilib']) + self.add_config(arch='m68k', + os_name='linux-gnu', + variant='coldfire', + gcc_cfg=['--with-arch=cf', '--disable-multilib']) + self.add_config(arch='microblaze', + os_name='linux-gnu', + gcc_cfg=['--disable-multilib']) + self.add_config(arch='microblazeel', + os_name='linux-gnu', + gcc_cfg=['--disable-multilib']) + self.add_config(arch='mips64', + os_name='linux-gnu', + gcc_cfg=['--with-mips-plt'], + glibcs=[{'variant': 'n32'}, + {'arch': 'mips', + 'ccopts': '-mabi=32'}, + {'variant': 'n64', + 'ccopts': '-mabi=64'}]) + self.add_config(arch='mips64', + os_name='linux-gnu', + variant='soft', + gcc_cfg=['--with-mips-plt', '--with-float=soft'], + glibcs=[{'variant': 'n32-soft', + 'cfg': ['--without-fp']}, + {'variant': 'soft', + 'arch': 'mips', + 'ccopts': '-mabi=32', + 'cfg': ['--without-fp']}, + {'variant': 'n64-soft', + 'ccopts': '-mabi=64', + 'cfg': ['--without-fp']}]) + self.add_config(arch='mips64', + os_name='linux-gnu', + variant='nan2008', + gcc_cfg=['--with-mips-plt', '--with-nan=2008', + '--with-arch-64=mips64r2', + '--with-arch-32=mips32r2'], + glibcs=[{'variant': 'n32-nan2008'}, + {'variant': 'nan2008', + 'arch': 'mips', + 'ccopts': '-mabi=32'}, + {'variant': 'n64-nan2008', + 'ccopts': '-mabi=64'}]) + self.add_config(arch='mips64', + os_name='linux-gnu', + variant='nan2008-soft', + gcc_cfg=['--with-mips-plt', '--with-nan=2008', + '--with-arch-64=mips64r2', + '--with-arch-32=mips32r2', + '--with-float=soft'], + glibcs=[{'variant': 'n32-nan2008-soft', + 'cfg': ['--without-fp']}, + {'variant': 'nan2008-soft', + 'arch': 'mips', + 'ccopts': '-mabi=32', + 'cfg': ['--without-fp']}, + {'variant': 'n64-nan2008-soft', + 'ccopts': '-mabi=64', + 'cfg': ['--without-fp']}]) + self.add_config(arch='mips64el', + os_name='linux-gnu', + gcc_cfg=['--with-mips-plt'], + glibcs=[{'variant': 'n32'}, + {'arch': 'mipsel', + 'ccopts': '-mabi=32'}, + {'variant': 'n64', + 'ccopts': '-mabi=64'}]) + self.add_config(arch='mips64el', + os_name='linux-gnu', + variant='soft', + gcc_cfg=['--with-mips-plt', '--with-float=soft'], + glibcs=[{'variant': 'n32-soft', + 'cfg': ['--without-fp']}, + {'variant': 'soft', + 'arch': 'mipsel', + 'ccopts': '-mabi=32', + 'cfg': ['--without-fp']}, + {'variant': 'n64-soft', + 'ccopts': '-mabi=64', + 'cfg': ['--without-fp']}]) + self.add_config(arch='mips64el', + os_name='linux-gnu', + variant='nan2008', + gcc_cfg=['--with-mips-plt', '--with-nan=2008', + '--with-arch-64=mips64r2', + '--with-arch-32=mips32r2'], + glibcs=[{'variant': 'n32-nan2008'}, + {'variant': 'nan2008', + 'arch': 'mipsel', + 'ccopts': '-mabi=32'}, + {'variant': 'n64-nan2008', + 'ccopts': '-mabi=64'}]) + self.add_config(arch='mips64el', + os_name='linux-gnu', + variant='nan2008-soft', + gcc_cfg=['--with-mips-plt', '--with-nan=2008', + '--with-arch-64=mips64r2', + '--with-arch-32=mips32r2', + '--with-float=soft'], + glibcs=[{'variant': 'n32-nan2008-soft', + 'cfg': ['--without-fp']}, + {'variant': 'nan2008-soft', + 'arch': 'mipsel', + 'ccopts': '-mabi=32', + 'cfg': ['--without-fp']}, + {'variant': 'n64-nan2008-soft', + 'ccopts': '-mabi=64', + 'cfg': ['--without-fp']}]) + self.add_config(arch='nios2', + os_name='linux-gnu') + self.add_config(arch='powerpc', + os_name='linux-gnu', + gcc_cfg=['--disable-multilib', '--enable-secureplt']) + self.add_config(arch='powerpc', + os_name='linux-gnu', + variant='soft', + gcc_cfg=['--disable-multilib', '--with-float=soft', + '--enable-secureplt'], + glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}]) + self.add_config(arch='powerpc64', + os_name='linux-gnu', + gcc_cfg=['--disable-multilib', '--enable-secureplt']) + self.add_config(arch='powerpc64le', + os_name='linux-gnu', + gcc_cfg=['--disable-multilib', '--enable-secureplt']) + self.add_config(arch='powerpc', + os_name='linux-gnuspe', + gcc_cfg=['--disable-multilib', '--enable-secureplt', + '--enable-e500-double'], + glibcs=[{'cfg': ['--without-fp']}]) + self.add_config(arch='powerpc', + os_name='linux-gnuspe', + variant='e500v1', + gcc_cfg=['--disable-multilib', '--enable-secureplt'], + glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}]) + self.add_config(arch='s390x', + os_name='linux-gnu', + glibcs=[{}, + {'arch': 's390', 'ccopts': '-m31'}]) + # SH is missing __builtin_trap support, so work around this; + # see <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216>. + no_isolate = ('-fno-isolate-erroneous-paths-dereference' + ' -fno-isolate-erroneous-paths-attribute') + self.add_config(arch='sh3', + os_name='linux-gnu', + glibcs=[{'ccopts': no_isolate}]) + self.add_config(arch='sh3eb', + os_name='linux-gnu', + glibcs=[{'ccopts': no_isolate}]) + self.add_config(arch='sh4', + os_name='linux-gnu', + glibcs=[{'ccopts': no_isolate}]) + self.add_config(arch='sh4eb', + os_name='linux-gnu', + glibcs=[{'ccopts': no_isolate}]) + self.add_config(arch='sh4', + os_name='linux-gnu', + variant='soft', + gcc_cfg=['--without-fp'], + glibcs=[{'variant': 'soft', + 'cfg': ['--without-fp'], + 'ccopts': no_isolate}]) + self.add_config(arch='sh4eb', + os_name='linux-gnu', + variant='soft', + gcc_cfg=['--without-fp'], + glibcs=[{'variant': 'soft', + 'cfg': ['--without-fp'], + 'ccopts': no_isolate}]) + self.add_config(arch='sparc64', + os_name='linux-gnu', + glibcs=[{}, + {'arch': 'sparcv9', + 'ccopts': '-m32 -mlong-double-128'}]) + self.add_config(arch='tilegx', + os_name='linux-gnu', + glibcs=[{}, + {'variant': '32', 'ccopts': '-m32'}]) + self.add_config(arch='tilegxbe', + os_name='linux-gnu', + glibcs=[{}, + {'variant': '32', 'ccopts': '-m32'}]) + self.add_config(arch='tilepro', + os_name='linux-gnu') + self.add_config(arch='x86_64', + os_name='linux-gnu', + gcc_cfg=['--with-multilib-list=m64,m32,mx32'], + glibcs=[{}, + {'variant': 'x32', 'ccopts': '-mx32'}, + {'arch': 'i686', 'ccopts': '-m32 -march=i686'}], + extra_glibcs=[{'variant': 'disable-multi-arch', + 'cfg': ['--disable-multi-arch']}, + {'variant': 'disable-multi-arch', + 'arch': 'i686', + 'ccopts': '-m32 -march=i686', + 'cfg': ['--disable-multi-arch']}, + {'arch': 'i486', + 'ccopts': '-m32 -march=i486'}, + {'arch': 'i586', + 'ccopts': '-m32 -march=i586'}]) + + def add_config(self, **args): + """Add an individual build configuration.""" + cfg = Config(self, **args) + if cfg.name in self.configs: + print('error: duplicate config %s' % cfg.name) + exit(1) + self.configs[cfg.name] = cfg + for c in cfg.all_glibcs: + if c.name in self.glibc_configs: + print('error: duplicate glibc config %s' % c.name) + exit(1) + self.glibc_configs[c.name] = c + + def component_srcdir(self, component): + """Return the source directory for a given component, e.g. gcc.""" + return os.path.join(self.srcdir, component) + + def component_builddir(self, action, config, component, subconfig=None): + """Return the directory to use for a build.""" + if config is None: + # Host libraries. + assert subconfig is None + return os.path.join(self.builddir, action, component) + if subconfig is None: + return os.path.join(self.builddir, action, config, component) + else: + # glibc build as part of compiler build. + return os.path.join(self.builddir, action, config, component, + subconfig) + + def compiler_installdir(self, config): + """Return the directory in which to install a compiler.""" + return os.path.join(self.installdir, 'compilers', config) + + def compiler_bindir(self, config): + """Return the directory in which to find compiler binaries.""" + return os.path.join(self.compiler_installdir(config), 'bin') + + def compiler_sysroot(self, config): + """Return the sysroot directory for a compiler.""" + return os.path.join(self.compiler_installdir(config), 'sysroot') + + def glibc_installdir(self, config): + """Return the directory in which to install glibc.""" + return os.path.join(self.installdir, 'glibcs', config) + + def run_builds(self, action, configs): + """Run the requested builds.""" + if action == 'checkout': + self.checkout(configs) + return + elif action == 'host-libraries': + if configs: + print('error: configurations specified for host-libraries') + exit(1) + self.build_host_libraries() + elif action == 'compilers': + self.build_compilers(configs) + else: + self.build_glibcs(configs) + self.write_files() + self.do_build() + + @staticmethod + def remove_dirs(*args): + """Remove directories and their contents if they exist.""" + for dir in args: + shutil.rmtree(dir, ignore_errors=True) + + @staticmethod + def remove_recreate_dirs(*args): + """Remove directories if they exist, and create them as empty.""" + Context.remove_dirs(*args) + for dir in args: + os.makedirs(dir, exist_ok=True) + + def add_makefile_cmdlist(self, target, cmdlist, logsdir): + """Add makefile text for a list of commands.""" + commands = cmdlist.makefile_commands(self.wrapper, logsdir) + self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' % + (target, target, target, commands)) + + def write_files(self): + """Write out the Makefile and wrapper script.""" + mftext = ''.join(self.makefile_pieces) + with open(self.makefile, 'w') as f: + f.write(mftext) + wrapper_text = ( + '#!/bin/sh\n' + 'prev_base=$1\n' + 'this_base=$2\n' + 'desc=$3\n' + 'dir=$4\n' + 'path=$5\n' + 'shift 5\n' + 'prev_status=$prev_base-status.txt\n' + 'this_status=$this_base-status.txt\n' + 'this_log=$this_base-log.txt\n' + 'date > "$this_log"\n' + 'echo >> "$this_log"\n' + 'echo "Description: $desc" >> "$this_log"\n' + 'echo "Command: $*" >> "$this_log"\n' + 'echo "Directory: $dir" >> "$this_log"\n' + 'echo "Path addition: $path" >> "$this_log"\n' + 'echo >> "$this_log"\n' + 'record_status ()\n' + '{\n' + ' echo >> "$this_log"\n' + ' echo "$1: $desc" > "$this_status"\n' + ' echo "$1: $desc" >> "$this_log"\n' + ' echo >> "$this_log"\n' + ' date >> "$this_log"\n' + ' echo "$1: $desc"\n' + ' exit 0\n' + '}\n' + 'check_error ()\n' + '{\n' + ' if [ "$1" != "0" ]; then\n' + ' record_status FAIL\n' + ' fi\n' + '}\n' + 'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n' + ' record_status UNRESOLVED\n' + 'fi\n' + 'if [ "$dir" ]; then\n' + ' cd "$dir"\n' + ' check_error "$?"\n' + 'fi\n' + 'if [ "$path" ]; then\n' + ' PATH=$path:$PATH\n' + 'fi\n' + '"$@" < /dev/null >> "$this_log" 2>&1\n' + 'check_error "$?"\n' + 'record_status PASS\n') + with open(self.wrapper, 'w') as f: + f.write(wrapper_text) + os.chmod(self.wrapper, + (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP| + stat.S_IROTH|stat.S_IXOTH)) + save_logs_text = ( + '#!/bin/sh\n' + 'if ! [ -f tests.sum ]; then\n' + ' echo "No test summary available."\n' + ' exit 0\n' + 'fi\n' + 'save_file ()\n' + '{\n' + ' echo "Contents of $1:"\n' + ' echo\n' + ' cat "$1"\n' + ' echo\n' + ' echo "End of contents of $1."\n' + ' echo\n' + '}\n' + 'save_file tests.sum\n' + 'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n' + 'for t in $non_pass_tests; do\n' + ' if [ -f "$t.out" ]; then\n' + ' save_file "$t.out"\n' + ' fi\n' + 'done\n') + with open(self.save_logs, 'w') as f: + f.write(save_logs_text) + os.chmod(self.save_logs, + (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP| + stat.S_IROTH|stat.S_IXOTH)) + + def do_build(self): + """Do the actual build.""" + cmd = ['make', '-j%d' % self.parallelism] + subprocess.run(cmd, cwd=self.builddir, check=True) + + def build_host_libraries(self): + """Build the host libraries.""" + installdir = self.host_libraries_installdir + builddir = os.path.join(self.builddir, 'host-libraries') + logsdir = os.path.join(self.logsdir, 'host-libraries') + self.remove_recreate_dirs(installdir, builddir, logsdir) + cmdlist = CommandList('host-libraries', self.keep) + self.build_host_library(cmdlist, 'gmp') + self.build_host_library(cmdlist, 'mpfr', + ['--with-gmp=%s' % installdir]) + self.build_host_library(cmdlist, 'mpc', + ['--with-gmp=%s' % installdir, + '--with-mpfr=%s' % installdir]) + cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')]) + self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir) + + def build_host_library(self, cmdlist, lib, extra_opts=None): + """Build one host library.""" + srcdir = self.component_srcdir(lib) + builddir = self.component_builddir('host-libraries', None, lib) + installdir = self.host_libraries_installdir + cmdlist.push_subdesc(lib) + cmdlist.create_use_dir(builddir) + cfg_cmd = [os.path.join(srcdir, 'configure'), + '--prefix=%s' % installdir, + '--disable-shared'] + if extra_opts: + cfg_cmd.extend (extra_opts) + cmdlist.add_command('configure', cfg_cmd) + cmdlist.add_command('build', ['make']) + cmdlist.add_command('check', ['make', 'check']) + cmdlist.add_command('install', ['make', 'install']) + cmdlist.cleanup_dir() + cmdlist.pop_subdesc() + + def build_compilers(self, configs): + """Build the compilers.""" + if not configs: + self.remove_dirs(os.path.join(self.builddir, 'compilers')) + self.remove_dirs(os.path.join(self.installdir, 'compilers')) + self.remove_dirs(os.path.join(self.logsdir, 'compilers')) + configs = sorted(self.configs.keys()) + for c in configs: + self.configs[c].build() + + def build_glibcs(self, configs): + """Build the glibcs.""" + if not configs: + self.remove_dirs(os.path.join(self.builddir, 'glibcs')) + self.remove_dirs(os.path.join(self.installdir, 'glibcs')) + self.remove_dirs(os.path.join(self.logsdir, 'glibcs')) + configs = sorted(self.glibc_configs.keys()) + for c in configs: + self.glibc_configs[c].build() + + def checkout(self, versions): + """Check out the desired component versions.""" + default_versions = {'binutils': 'vcs-2.27', + 'gcc': 'vcs-6', + 'glibc': 'vcs-mainline', + 'gmp': '6.1.1', + 'linux': '4.8.6', + 'mpc': '1.0.3', + 'mpfr': '3.1.5'} + use_versions = {} + for v in versions: + found_v = False + for k in default_versions.keys(): + kx = k + '-' + if v.startswith(kx): + vx = v[len(kx):] + if k in use_versions: + print('error: multiple versions for %s' % k) + exit(1) + use_versions[k] = vx + found_v = True + break + if not found_v: + print('error: unknown component in %s' % v) + exit(1) + for k in default_versions.keys(): + if k not in use_versions: + use_versions[k] = default_versions[k] + os.makedirs(self.srcdir, exist_ok=True) + for k in sorted(default_versions.keys()): + update = os.access(self.component_srcdir(k), os.F_OK) + v = use_versions[k] + if v.startswith('vcs-'): + self.checkout_vcs(k, v[4:], update) + else: + self.checkout_tar(k, v, update) + + def checkout_vcs(self, component, version, update): + """Check out the given version of the given component from version + control.""" + if component == 'binutils': + git_url = 'git://sourceware.org/git/binutils-gdb.git'< |