Simplify package installation on CI
Having a Python script was a good idea back when we could do almost the same steps on every CI configuration. This turned out to grow into a complicated script, so it's easier to split off things for Linux/OS X into a small shell script (and keep Python for Windows, as I really don't want to use .bat/.ps).
This commit is contained in:
Florian Bruhin
parent
768ff8b193
commit
6558e196b4
@ -11,7 +11,7 @@ environment:
|
||||
- TESTENV: pylint
|
||||
|
||||
install:
|
||||
- C:\Python27\python -u scripts\dev\ci\install.py
|
||||
- C:\Python27\python -u scripts\dev\ci\appveyor_install.py
|
||||
|
||||
test_script:
|
||||
- C:\Python34\Scripts\tox -e %TESTENV%
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
sudo: required
|
||||
dist: trusty
|
||||
# Not really, but this is here so we can do stuff by hand.
|
||||
language: c
|
||||
language: python
|
||||
|
||||
matrix:
|
||||
include:
|
||||
@ -48,7 +47,7 @@ before_install:
|
||||
- 'export PATH="/usr/bin:$PATH"'
|
||||
|
||||
install:
|
||||
- python scripts/dev/ci/install.py
|
||||
- bash scripts/dev/ci/travis_install.sh
|
||||
|
||||
script:
|
||||
- bash scripts/dev/ci/travis_run.sh
|
||||
|
||||
72
scripts/dev/ci/appveyor_install.py
Normal file
72
scripts/dev/ci/appveyor_install.py
Normal file
@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python2
|
||||
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
|
||||
|
||||
# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
|
||||
|
||||
# This file is part of qutebrowser.
|
||||
#
|
||||
# qutebrowser is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# qutebrowser is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# pylint: skip-file
|
||||
|
||||
"""Install needed prerequisites on the AppVeyor.
|
||||
|
||||
Note this file is written in python2 as this is more readily available on the
|
||||
CI machines.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import subprocess
|
||||
import urllib
|
||||
|
||||
try:
|
||||
import _winreg as winreg
|
||||
except ImportError:
|
||||
winreg = None
|
||||
|
||||
|
||||
def check_setup(executable):
|
||||
subprocess.check_call([executable, '-c', 'import PyQt5'])
|
||||
subprocess.check_call([executable, '-c', 'import sip'])
|
||||
subprocess.check_call([executable, '--version'])
|
||||
|
||||
|
||||
print("Getting PyQt5...")
|
||||
qt_version = '5.5.1'
|
||||
pyqt_version = '5.5.1'
|
||||
pyqt_url = ('http://www.qutebrowser.org/pyqt/'
|
||||
'PyQt5-{}-gpl-Py3.4-Qt{}-x32.exe'.format(
|
||||
pyqt_version, qt_version))
|
||||
urllib.urlretrieve(pyqt_url, r'C:\install-PyQt5.exe')
|
||||
|
||||
print("Fixing registry...")
|
||||
with winreg.OpenKey(winreg.HKEY_CURRENT_USER,
|
||||
r'Software\Python\PythonCore\3.4', 0,
|
||||
winreg.KEY_WRITE) as key:
|
||||
winreg.SetValue(key, 'InstallPath', winreg.REG_SZ, r'C:\Python34')
|
||||
|
||||
print("Installing PyQt5...")
|
||||
subprocess.check_call([r'C:\install-PyQt5.exe', '/S'])
|
||||
|
||||
print("Installing pip/tox")
|
||||
subprocess.check_call([r'C:\Python34\python', '-m', 'pip', 'install', '-U',
|
||||
'pip'])
|
||||
subprocess.check_call([r'C:\Python34\Scripts\pip', 'install', '-U', 'tox'])
|
||||
|
||||
print("Linking Python...")
|
||||
with open(r'C:\Windows\system32\python3.bat', 'w') as f:
|
||||
f.write(r'@C:\Python34\python %*')
|
||||
|
||||
check_setup(r'C:\Python34\python')
|
||||
@ -1,174 +0,0 @@
|
||||
#!/usr/bin/env python2
|
||||
# vim: ft=python fileencoding=utf-8 sts=4 sw=4 et:
|
||||
|
||||
# Copyright 2015-2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
|
||||
|
||||
# This file is part of qutebrowser.
|
||||
#
|
||||
# qutebrowser is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# qutebrowser is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# pylint: skip-file
|
||||
|
||||
"""Install needed prerequisites on the AppVeyor/Travis CI.
|
||||
|
||||
Note this file is written in python2 as this is more readily available on the
|
||||
CI machines.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import urllib
|
||||
import contextlib
|
||||
import time
|
||||
|
||||
try:
|
||||
import _winreg as winreg
|
||||
except ImportError:
|
||||
winreg = None
|
||||
|
||||
TESTENV = os.environ.get('TESTENV', None)
|
||||
TRAVIS_OS = os.environ.get('TRAVIS_OS_NAME', None)
|
||||
INSTALL_PYQT = TESTENV in ('py34', 'py35', 'py34-cov', 'py35-cov',
|
||||
'unittests-nodisp', 'vulture', 'pylint', 'docs')
|
||||
XVFB = TRAVIS_OS == 'linux' and TESTENV == 'py34'
|
||||
pip_packages = ['tox']
|
||||
if TESTENV is not None and TESTENV.endswith('-cov'):
|
||||
pip_packages.append('codecov')
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def travis_fold(text):
|
||||
if 'TRAVIS' in os.environ:
|
||||
marker = re.compile(r'\W+').sub('-', text.lower()).strip('-')
|
||||
print("travis_fold:start:{}".format(marker))
|
||||
yield
|
||||
print("travis_fold:end:{}".format(marker))
|
||||
else:
|
||||
yield
|
||||
|
||||
|
||||
def folded_cmd(argv):
|
||||
"""Output a command with travis folding markers."""
|
||||
with travis_fold(''.join(argv)):
|
||||
print(" $ " + ' '.join(argv))
|
||||
subprocess.check_call(argv)
|
||||
|
||||
|
||||
def apt_get(args):
|
||||
try:
|
||||
folded_cmd(['sudo', 'apt-get', '-y', '-q'] + args)
|
||||
except subprocess.CalledProcessError:
|
||||
print()
|
||||
print("apt-get failed... trying a second time in 30s...")
|
||||
print()
|
||||
time.sleep(30)
|
||||
folded_cmd(['sudo', 'apt-get', '-y', '-q'] + args)
|
||||
|
||||
|
||||
def brew(args):
|
||||
folded_cmd(['brew'] + args)
|
||||
|
||||
|
||||
def check_setup(executable):
|
||||
if INSTALL_PYQT:
|
||||
print("Checking setup...")
|
||||
subprocess.check_call([executable, '-c', 'import PyQt5'])
|
||||
subprocess.check_call([executable, '-c', 'import sip'])
|
||||
subprocess.check_call([executable, '--version'])
|
||||
|
||||
|
||||
if 'APPVEYOR' in os.environ:
|
||||
print("Getting PyQt5...")
|
||||
qt_version = '5.5.1'
|
||||
pyqt_version = '5.5.1'
|
||||
pyqt_url = ('http://www.qutebrowser.org/pyqt/'
|
||||
'PyQt5-{}-gpl-Py3.4-Qt{}-x32.exe'.format(
|
||||
pyqt_version, qt_version))
|
||||
urllib.urlretrieve(pyqt_url, r'C:\install-PyQt5.exe')
|
||||
|
||||
print("Fixing registry...")
|
||||
with winreg.OpenKey(winreg.HKEY_CURRENT_USER,
|
||||
r'Software\Python\PythonCore\3.4', 0,
|
||||
winreg.KEY_WRITE) as key:
|
||||
winreg.SetValue(key, 'InstallPath', winreg.REG_SZ, r'C:\Python34')
|
||||
|
||||
print("Installing PyQt5...")
|
||||
subprocess.check_call([r'C:\install-PyQt5.exe', '/S'])
|
||||
|
||||
folded_cmd([r'C:\Python34\python', '-m', 'pip', 'install', '-U', 'pip'])
|
||||
folded_cmd([r'C:\Python34\Scripts\pip', 'install', '-U'] + pip_packages)
|
||||
|
||||
print("Linking Python...")
|
||||
with open(r'C:\Windows\system32\python3.bat', 'w') as f:
|
||||
f.write(r'@C:\Python34\python %*')
|
||||
|
||||
check_setup(r'C:\Python34\python')
|
||||
elif TRAVIS_OS == 'linux' and 'DOCKER' in os.environ:
|
||||
pass
|
||||
elif TRAVIS_OS == 'linux':
|
||||
folded_cmd(['sudo', '-H', 'pip', 'install', '-U', 'pip'])
|
||||
folded_cmd(['sudo', '-H', 'pip', 'install', '-U'] + pip_packages)
|
||||
|
||||
pkgs = []
|
||||
|
||||
if XVFB:
|
||||
pkgs.append('xvfb')
|
||||
if INSTALL_PYQT:
|
||||
pkgs += ['python3-pyqt5', 'python3-pyqt5.qtwebkit']
|
||||
if TESTENV == 'eslint':
|
||||
pkgs += ['npm', 'nodejs', 'nodejs-legacy']
|
||||
if TESTENV == 'docs':
|
||||
pkgs += ['asciidoc']
|
||||
|
||||
if pkgs:
|
||||
apt_get(['update'])
|
||||
apt_get(['install', '--no-install-recommends'] + pkgs)
|
||||
|
||||
if TESTENV == 'flake8':
|
||||
apt_get(['update'])
|
||||
# We need an up-to-date Python because of:
|
||||
# https://github.com/google/yapf/issues/46
|
||||
apt_get(['install', '-t', 'trusty-updates', 'python3.4'])
|
||||
|
||||
if TESTENV == 'eslint':
|
||||
folded_cmd(['sudo', 'npm', 'install', '-g', 'npm'])
|
||||
folded_cmd(['sudo', 'npm', 'install', '-g', 'eslint'])
|
||||
else:
|
||||
check_setup('python3')
|
||||
elif TRAVIS_OS == 'osx':
|
||||
print("Disabling App Nap...")
|
||||
subprocess.check_call(['defaults', 'write', 'NSGlobalDomain',
|
||||
'NSAppSleepDisabled', '-bool', 'YES'])
|
||||
brew(['update'])
|
||||
|
||||
pkgs = ['python3']
|
||||
if INSTALL_PYQT:
|
||||
pkgs.append('pyqt5')
|
||||
brew(['install', '--verbose'] + pkgs)
|
||||
|
||||
folded_cmd(['sudo', 'pip3', 'install'] + pip_packages)
|
||||
|
||||
check_setup('python3')
|
||||
else:
|
||||
def env(key):
|
||||
return os.environ.get(key, None)
|
||||
print("Unknown environment! (CI {}, APPVEYOR {}, TRAVIS {}, "
|
||||
"TRAVIS_OS_NAME {})".format(env('CI'), env('APPVEYOR'),
|
||||
env('TRAVIS'), env('TRAVIS_OS_NAME')),
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
128
scripts/dev/ci/travis_install.sh
Normal file
128
scripts/dev/ci/travis_install.sh
Normal file
@ -0,0 +1,128 @@
|
||||
# vim: ft=sh fileencoding=utf-8 sts=4 sw=4 et:
|
||||
|
||||
# Copyright 2016 Florian Bruhin (The Compiler) <mail@qutebrowser.org>
|
||||
|
||||
# This file is part of qutebrowser.
|
||||
#
|
||||
# qutebrowser is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# qutebrowser is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with qutebrowser. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
# Stolen from https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/templates/header.sh
|
||||
travis_retry() {
|
||||
local ANSI_RED="\033[31;1m"
|
||||
local ANSI_RESET="\033[0m"
|
||||
local result=0
|
||||
local count=1
|
||||
while [ $count -le 3 ]; do
|
||||
[ $result -ne 0 ] && {
|
||||
echo -e "\n${ANSI_RED}The command \"$@\" failed. Retrying, $count of 3.${ANSI_RESET}\n" >&2
|
||||
}
|
||||
"$@"
|
||||
result=$?
|
||||
[ $result -eq 0 ] && break
|
||||
count=$(($count + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
[ $count -gt 3 ] && {
|
||||
echo -e "\n${ANSI_RED}The command \"$@\" failed 3 times.${ANSI_RESET}\n" >&2
|
||||
}
|
||||
|
||||
return $result
|
||||
}
|
||||
|
||||
apt_install() {
|
||||
travis_retry sudo apt-get -y -q update
|
||||
travis_retry sudo apt-get -y -q install --no-install-recommends "$@"
|
||||
}
|
||||
|
||||
brew_install() {
|
||||
brew update
|
||||
brew install "$@"
|
||||
}
|
||||
|
||||
pip_install() {
|
||||
# Make sure pip is up-to-date first
|
||||
travis_retry sudo -H python3 -m pip install -U pip
|
||||
travis_retry sudo -H python3 -m pip install -U "$@"
|
||||
}
|
||||
|
||||
npm_install() {
|
||||
# Make sure npm is up-to-date first
|
||||
travis_retry sudo npm install -g npm
|
||||
travis_retry sudo npm install -g "$@"
|
||||
}
|
||||
|
||||
check_pyqt() {
|
||||
python3 <<EOF
|
||||
import sys
|
||||
from PyQt5.QtCore import QT_VERSION_STR, PYQT_VERSION_STR, qVersion
|
||||
from sip import SIP_VERSION_STR
|
||||
|
||||
print("Python {}".format(sys.version))
|
||||
print("PyQt5 {}".format(PYQT_VERSION_STR))
|
||||
print("Qt5 {} (runtime {})".format(QT_VERSION_STR, qVersion()))
|
||||
print("sip {}".format(SIP_VERSION_STR))
|
||||
EOF
|
||||
}
|
||||
|
||||
set -e
|
||||
|
||||
if [[ $DOCKER ]]; then
|
||||
exit 0
|
||||
elif [[ $TRAVIS_OS_NAME == osx ]]; then
|
||||
# Disable App Nap
|
||||
defaults write NSGlobalDomain NSAppSleepDisabled -bool YES
|
||||
brew_install python3 pyqt5
|
||||
pip_install tox
|
||||
check_pyqt
|
||||
exit 0
|
||||
fi
|
||||
|
||||
pyqt_pkgs="python3-pyqt5 python3-pyqt5.qtwebkit"
|
||||
|
||||
case $TESTENV in
|
||||
py34-cov)
|
||||
apt_install python3-pip xvfb $pyqt_pkgs
|
||||
pip_install tox codecov
|
||||
check_pyqt
|
||||
;;
|
||||
pylint|vulture)
|
||||
apt_install python3-pip $pyqt_pkgs
|
||||
pip_install tox
|
||||
check_pyqt
|
||||
;;
|
||||
flake8)
|
||||
# We need an up-to-date Python because of:
|
||||
# https://github.com/google/yapf/issues/46
|
||||
apt_install -t trusty-updates python3.4 python3-pip
|
||||
pip_install tox
|
||||
;;
|
||||
docs)
|
||||
apt_install python3-pip $pyqt_pkgs asciidoc
|
||||
pip_install tox
|
||||
check_pyqt
|
||||
;;
|
||||
misc|pyroma|check-manifest)
|
||||
pip_install tox
|
||||
;;
|
||||
eslint)
|
||||
apt_install python3-pip npm nodejs nodejs-legacy
|
||||
pip_install tox
|
||||
npm_install eslint
|
||||
;;
|
||||
*)
|
||||
echo "Unknown testenv $TESTENV!" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Loading…
Reference in New Issue
Block a user