From 6558e196b484c5de9b072dd239f490228563aebc Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Wed, 11 May 2016 15:41:57 +0200 Subject: [PATCH] Simplify package installation on CI Having a Python script was a good idea back when we could do almost the same steps on every CI configuration. This turned out to grow into a complicated script, so it's easier to split off things for Linux/OS X into a small shell script (and keep Python for Windows, as I really don't want to use .bat/.ps). --- .appveyor.yml | 2 +- .travis.yml | 5 +- scripts/dev/ci/appveyor_install.py | 72 ++++++++++++ scripts/dev/ci/install.py | 174 ----------------------------- scripts/dev/ci/travis_install.sh | 128 +++++++++++++++++++++ 5 files changed, 203 insertions(+), 178 deletions(-) create mode 100644 scripts/dev/ci/appveyor_install.py delete mode 100644 scripts/dev/ci/install.py create mode 100644 scripts/dev/ci/travis_install.sh diff --git a/.appveyor.yml b/.appveyor.yml index b85b64285..0f970d98b 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,7 +11,7 @@ environment: - TESTENV: pylint install: - - C:\Python27\python -u scripts\dev\ci\install.py + - C:\Python27\python -u scripts\dev\ci\appveyor_install.py test_script: - C:\Python34\Scripts\tox -e %TESTENV% diff --git a/.travis.yml b/.travis.yml index e6608ac15..14f73cc49 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ sudo: required dist: trusty -# Not really, but this is here so we can do stuff by hand. -language: c +language: python matrix: include: @@ -48,7 +47,7 @@ before_install: - 'export PATH="/usr/bin:$PATH"' install: - - python scripts/dev/ci/install.py + - bash scripts/dev/ci/travis_install.sh script: - bash scripts/dev/ci/travis_run.sh diff --git a/scripts/dev/ci/appveyor_install.py b/scripts/dev/ci/appveyor_install.py new file mode 100644 index 000000000..e78ac36dd --- /dev/null +++ b/scripts/dev/ci/appveyor_install.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python2 +# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2015-2016 Florian Bruhin (The Compiler) + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +# pylint: skip-file + +"""Install needed prerequisites on the AppVeyor. + +Note this file is written in python2 as this is more readily available on the +CI machines. +""" + +from __future__ import print_function + +import subprocess +import urllib + +try: + import _winreg as winreg +except ImportError: + winreg = None + + +def check_setup(executable): + subprocess.check_call([executable, '-c', 'import PyQt5']) + subprocess.check_call([executable, '-c', 'import sip']) + subprocess.check_call([executable, '--version']) + + +print("Getting PyQt5...") +qt_version = '5.5.1' +pyqt_version = '5.5.1' +pyqt_url = ('http://www.qutebrowser.org/pyqt/' + 'PyQt5-{}-gpl-Py3.4-Qt{}-x32.exe'.format( + pyqt_version, qt_version)) +urllib.urlretrieve(pyqt_url, r'C:\install-PyQt5.exe') + +print("Fixing registry...") +with winreg.OpenKey(winreg.HKEY_CURRENT_USER, + r'Software\Python\PythonCore\3.4', 0, + winreg.KEY_WRITE) as key: + winreg.SetValue(key, 'InstallPath', winreg.REG_SZ, r'C:\Python34') + +print("Installing PyQt5...") +subprocess.check_call([r'C:\install-PyQt5.exe', '/S']) + +print("Installing pip/tox") +subprocess.check_call([r'C:\Python34\python', '-m', 'pip', 'install', '-U', + 'pip']) +subprocess.check_call([r'C:\Python34\Scripts\pip', 'install', '-U', 'tox']) + +print("Linking Python...") +with open(r'C:\Windows\system32\python3.bat', 'w') as f: + f.write(r'@C:\Python34\python %*') + +check_setup(r'C:\Python34\python') diff --git a/scripts/dev/ci/install.py b/scripts/dev/ci/install.py deleted file mode 100644 index 9979e7949..000000000 --- a/scripts/dev/ci/install.py +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/env python2 -# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et: - -# Copyright 2015-2016 Florian Bruhin (The Compiler) - -# This file is part of qutebrowser. -# -# qutebrowser is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# qutebrowser is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with qutebrowser. If not, see . - -# pylint: skip-file - -"""Install needed prerequisites on the AppVeyor/Travis CI. - -Note this file is written in python2 as this is more readily available on the -CI machines. -""" - -from __future__ import print_function - -import os -import re -import sys -import subprocess -import urllib -import contextlib -import time - -try: - import _winreg as winreg -except ImportError: - winreg = None - -TESTENV = os.environ.get('TESTENV', None) -TRAVIS_OS = os.environ.get('TRAVIS_OS_NAME', None) -INSTALL_PYQT = TESTENV in ('py34', 'py35', 'py34-cov', 'py35-cov', - 'unittests-nodisp', 'vulture', 'pylint', 'docs') -XVFB = TRAVIS_OS == 'linux' and TESTENV == 'py34' -pip_packages = ['tox'] -if TESTENV is not None and TESTENV.endswith('-cov'): - pip_packages.append('codecov') - - -@contextlib.contextmanager -def travis_fold(text): - if 'TRAVIS' in os.environ: - marker = re.compile(r'\W+').sub('-', text.lower()).strip('-') - print("travis_fold:start:{}".format(marker)) - yield - print("travis_fold:end:{}".format(marker)) - else: - yield - - -def folded_cmd(argv): - """Output a command with travis folding markers.""" - with travis_fold(''.join(argv)): - print(" $ " + ' '.join(argv)) - subprocess.check_call(argv) - - -def apt_get(args): - try: - folded_cmd(['sudo', 'apt-get', '-y', '-q'] + args) - except subprocess.CalledProcessError: - print() - print("apt-get failed... trying a second time in 30s...") - print() - time.sleep(30) - folded_cmd(['sudo', 'apt-get', '-y', '-q'] + args) - - -def brew(args): - folded_cmd(['brew'] + args) - - -def check_setup(executable): - if INSTALL_PYQT: - print("Checking setup...") - subprocess.check_call([executable, '-c', 'import PyQt5']) - subprocess.check_call([executable, '-c', 'import sip']) - subprocess.check_call([executable, '--version']) - - -if 'APPVEYOR' in os.environ: - print("Getting PyQt5...") - qt_version = '5.5.1' - pyqt_version = '5.5.1' - pyqt_url = ('http://www.qutebrowser.org/pyqt/' - 'PyQt5-{}-gpl-Py3.4-Qt{}-x32.exe'.format( - pyqt_version, qt_version)) - urllib.urlretrieve(pyqt_url, r'C:\install-PyQt5.exe') - - print("Fixing registry...") - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, - r'Software\Python\PythonCore\3.4', 0, - winreg.KEY_WRITE) as key: - winreg.SetValue(key, 'InstallPath', winreg.REG_SZ, r'C:\Python34') - - print("Installing PyQt5...") - subprocess.check_call([r'C:\install-PyQt5.exe', '/S']) - - folded_cmd([r'C:\Python34\python', '-m', 'pip', 'install', '-U', 'pip']) - folded_cmd([r'C:\Python34\Scripts\pip', 'install', '-U'] + pip_packages) - - print("Linking Python...") - with open(r'C:\Windows\system32\python3.bat', 'w') as f: - f.write(r'@C:\Python34\python %*') - - check_setup(r'C:\Python34\python') -elif TRAVIS_OS == 'linux' and 'DOCKER' in os.environ: - pass -elif TRAVIS_OS == 'linux': - folded_cmd(['sudo', '-H', 'pip', 'install', '-U', 'pip']) - folded_cmd(['sudo', '-H', 'pip', 'install', '-U'] + pip_packages) - - pkgs = [] - - if XVFB: - pkgs.append('xvfb') - if INSTALL_PYQT: - pkgs += ['python3-pyqt5', 'python3-pyqt5.qtwebkit'] - if TESTENV == 'eslint': - pkgs += ['npm', 'nodejs', 'nodejs-legacy'] - if TESTENV == 'docs': - pkgs += ['asciidoc'] - - if pkgs: - apt_get(['update']) - apt_get(['install', '--no-install-recommends'] + pkgs) - - if TESTENV == 'flake8': - apt_get(['update']) - # We need an up-to-date Python because of: - # https://github.com/google/yapf/issues/46 - apt_get(['install', '-t', 'trusty-updates', 'python3.4']) - - if TESTENV == 'eslint': - folded_cmd(['sudo', 'npm', 'install', '-g', 'npm']) - folded_cmd(['sudo', 'npm', 'install', '-g', 'eslint']) - else: - check_setup('python3') -elif TRAVIS_OS == 'osx': - print("Disabling App Nap...") - subprocess.check_call(['defaults', 'write', 'NSGlobalDomain', - 'NSAppSleepDisabled', '-bool', 'YES']) - brew(['update']) - - pkgs = ['python3'] - if INSTALL_PYQT: - pkgs.append('pyqt5') - brew(['install', '--verbose'] + pkgs) - - folded_cmd(['sudo', 'pip3', 'install'] + pip_packages) - - check_setup('python3') -else: - def env(key): - return os.environ.get(key, None) - print("Unknown environment! (CI {}, APPVEYOR {}, TRAVIS {}, " - "TRAVIS_OS_NAME {})".format(env('CI'), env('APPVEYOR'), - env('TRAVIS'), env('TRAVIS_OS_NAME')), - file=sys.stderr) - sys.exit(1) diff --git a/scripts/dev/ci/travis_install.sh b/scripts/dev/ci/travis_install.sh new file mode 100644 index 000000000..a38ee3731 --- /dev/null +++ b/scripts/dev/ci/travis_install.sh @@ -0,0 +1,128 @@ +# vim: ft=sh fileencoding=utf-8 sts=4 sw=4 et: + +# Copyright 2016 Florian Bruhin (The Compiler) + +# This file is part of qutebrowser. +# +# qutebrowser is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# qutebrowser is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with qutebrowser. If not, see . + +# Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh +travis_retry() { + local ANSI_RED="\033[31;1m" + local ANSI_RESET="\033[0m" + local result=0 + local count=1 + while [ $count -le 3 ]; do + [ $result -ne 0 ] && { + echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2 + } + "$@" + result=$? + [ $result -eq 0 ] && break + count=$(($count + 1)) + sleep 1 + done + + [ $count -gt 3 ] && { + echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2 + } + + return $result +} + +apt_install() { + travis_retry sudo apt-get -y -q update + travis_retry sudo apt-get -y -q install --no-install-recommends "$@" +} + +brew_install() { + brew update + brew install "$@" +} + +pip_install() { + # Make sure pip is up-to-date first + travis_retry sudo -H python3 -m pip install -U pip + travis_retry sudo -H python3 -m pip install -U "$@" +} + +npm_install() { + # Make sure npm is up-to-date first + travis_retry sudo npm install -g npm + travis_retry sudo npm install -g "$@" +} + +check_pyqt() { + python3 <&2 + exit 1 + ;; +esac