From 4eefc53ed051a9b15feb074e7c949bb2c2a993b4 Mon Sep 17 00:00:00 2001 From: Patric Schmitz Date: Fri, 9 Jan 2015 02:24:48 +0100 Subject: [PATCH 01/11] Change init_venv to use python 3.x venv --force was replaced in favor of --clear and --upgrade which correspond to the respective pyvenv options. The pyvenv help is not explicit on the behavior if --clear is not given but the path exists. https://docs.python.org/3/library/venv.html states pyvenv would fail in that case, but it does not with Python 3.4.2, which I don't consider a problem however. Added a newline here and there for better readability. --- scripts/init_venv.py | 54 ++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index 89de817aa..adb4d108e 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -18,7 +18,7 @@ # You should have received a copy of the GNU General Public License # along with qutebrowser. If not, see . -"""Initialize a virtualenv suitable to be used for qutebrowser.""" +"""Initialize a venv suitable to be used for qutebrowser.""" import os import re @@ -42,28 +42,19 @@ g_args = None def parse_args(): """Parse the commandline arguments.""" parser = argparse.ArgumentParser() - parser.add_argument('--force', help="Force creating a new virtualenv.", + parser.add_argument('--clear', help="Clear venv in case it already exists.", + action='store_true') + parser.add_argument('--upgrade', help="Upgrade venv to use this version of " + "Python, assuming Python has been upgraded in-place.", action='store_true') parser.add_argument('--dev', help="Set up an environment suitable for " - "developing qutebrowser.", + "developing qutebrowser.", action='store_true') - parser.add_argument('path', help="Path to the virtualenv folder", + parser.add_argument('path', help="Path to the venv folder", default='.venv', nargs='?') return parser.parse_args() -def check_exists(): - """Check if the virtualenv already exists.""" - if os.path.exists(g_path): - if g_args.force: - print("Deleting old virtualenv at {}".format(g_path)) - shutil.rmtree(g_path) - else: - print("virtualenv at {} does already exist!".format(g_path), - file=sys.stderr) - sys.exit(1) - - def get_dev_packages(short=False): """Get a list of packages to install. @@ -85,7 +76,7 @@ def install_dev_packages(): def venv_python(*args, output=False): - """Call the virtualenv's python with the given arguments.""" + """Call the venv's python with the given arguments.""" subdir = 'Scripts' if os.name == 'nt' else 'bin' executable = os.path.join(g_path, subdir, os.path.basename(sys.executable)) env = dict(os.environ) @@ -102,6 +93,7 @@ def venv_python(*args, output=False): def test_toolchain(): """Test if imports work properly.""" utils.print_title("Checking toolchain") + packages = ['sip', 'PyQt5.QtCore', 'PyQt5.QtWebKit', 'qutebrowser.app'] if g_args.dev: packages += get_dev_packages(short=True) @@ -113,19 +105,22 @@ def test_toolchain(): def link_pyqt(): - """Symlink the systemwide PyQt/sip into the virtualenv.""" + """Symlink the systemwide PyQt/sip into the venv.""" if os.name == 'nt': return utils.print_title("Softlinking PyQt5") + sys_path = distutils.sysconfig.get_python_lib() venv_path = venv_python( '-c', 'from distutils.sysconfig import get_python_lib\n' 'print(get_python_lib())', output=True).rstrip() + globbed_sip = (glob.glob(os.path.join(sys_path, 'sip*.so')) + glob.glob(os.path.join(sys_path, 'sip*.pyd'))) if not globbed_sip: print("Did not find sip in {}!".format(sys_path), file=sys.stderr) sys.exit(1) + files = [ 'PyQt5', ] @@ -135,19 +130,28 @@ def link_pyqt(): link_name = os.path.join(venv_path, fn) if not os.path.exists(source): raise FileNotFoundError(source) + if os.path.exists(link_name): + os.unlink(link_name) print('{} -> {}'.format(source, link_name)) os.symlink(source, link_name) def create_venv(): - """Create a new virtualenv.""" - utils.print_title("Creating virtualenv") + """Create a new venv.""" + utils.print_title("Creating venv using pyvenv") + if os.name == 'nt': sys_site = ['--system-site-packages'] else: sys_site = [] - subprocess.check_call(['virtualenv'] + sys_site + - ['-p', sys.executable, g_path]) + + command = ['pyvenv'] + sys_site + if g_args.clear: + command += ['--clear'] + if g_args.upgrade: + command += ['--upgrade'] + command += [g_path] + subprocess.check_call(command) def main(): @@ -158,10 +162,12 @@ def main(): print("Refusing to run with empty path!", file=sys.stderr) sys.exit(1) g_path = os.path.abspath(g_args.path) - check_exists() + create_venv() - utils.print_title("Calling setup.py") + + utils.print_title("Calling: setup.py develop") venv_python('setup.py', 'develop') + if g_args.dev: utils.print_title("Installing developer packages") install_dev_packages() From 7813d9a93d6f4310e67f49638e27d08abb2a52c3 Mon Sep 17 00:00:00 2001 From: Patric Schmitz Date: Fri, 9 Jan 2015 15:43:34 +0100 Subject: [PATCH 02/11] Add hidden --force option for backwards compatibility with existing scripts --- scripts/init_venv.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index adb4d108e..fd8e549e2 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -47,6 +47,8 @@ def parse_args(): parser.add_argument('--upgrade', help="Upgrade venv to use this version of " "Python, assuming Python has been upgraded in-place.", action='store_true') + parser.add_argument('--force', help=argparse.SUPPRESS, + action='store_true') parser.add_argument('--dev', help="Set up an environment suitable for " "developing qutebrowser.", action='store_true') @@ -146,7 +148,7 @@ def create_venv(): sys_site = [] command = ['pyvenv'] + sys_site - if g_args.clear: + if g_args.clear or g_args.force: command += ['--clear'] if g_args.upgrade: command += ['--upgrade'] From 2404c75012b1064ab5fb30a764cb342b786290fd Mon Sep 17 00:00:00 2001 From: Patric Schmitz Date: Fri, 9 Jan 2015 18:46:11 +0100 Subject: [PATCH 03/11] Use venv API for building the venv instead of calling pyvenv --- scripts/init_venv.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index fd8e549e2..bdacd09cf 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -30,6 +30,7 @@ import argparse import subprocess import distutils.sysconfig # pylint: disable=import-error # see https://bitbucket.org/logilab/pylint/issue/73/ +import venv sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir)) from scripts import utils @@ -140,21 +141,29 @@ def link_pyqt(): def create_venv(): """Create a new venv.""" - utils.print_title("Creating venv using pyvenv") + utils.print_title("Creating venv") - if os.name == 'nt': - sys_site = ['--system-site-packages'] - else: - sys_site = [] + system_site_packages = True - command = ['pyvenv'] + sys_site + clear = False if g_args.clear or g_args.force: - command += ['--clear'] - if g_args.upgrade: - command += ['--upgrade'] - command += [g_path] - subprocess.check_call(command) + clear = True + upgrade = False + if g_args.upgrade: + upgrade = True + + symlinks = False if os.name == 'nt' else True + + builder = venv.EnvBuilder(system_site_packages=system_site_packages, + clear=clear, upgrade=upgrade, + symlinks=symlinks, with_pip=True) + + context = builder.ensure_directories(g_path) + builder.create_configuration(context) + builder.setup_python(context) + builder.setup_scripts(context) + def main(): """Main entry point.""" From f76ce3c152fa28d500bf03aa27318b72a68c9d80 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 11:51:40 +0100 Subject: [PATCH 04/11] Regenerate authors See #463. --- README.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/README.asciidoc b/README.asciidoc index 8bfee7fbb..e6579115d 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -132,6 +132,7 @@ Contributors, sorted by the number of commits in descending order: * Claude * Peter Vilim * John ShaggyTwoDope Jenkins +* Patric Schmitz * rikn00 * Martin Zimmermann * Joel Torstensson From 816fcf3a6c14df61498260217d9818fe7762addb Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 11:53:18 +0100 Subject: [PATCH 05/11] Fix whitespace at EOL. See #463. --- scripts/init_venv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index bdacd09cf..edb2f2afb 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -154,7 +154,7 @@ def create_venv(): upgrade = True symlinks = False if os.name == 'nt' else True - + builder = venv.EnvBuilder(system_site_packages=system_site_packages, clear=clear, upgrade=upgrade, symlinks=symlinks, with_pip=True) @@ -163,7 +163,7 @@ def create_venv(): builder.create_configuration(context) builder.setup_python(context) builder.setup_scripts(context) - + def main(): """Main entry point.""" From 2264b55e92eef2823d468dc86c5b103ca71f2f64 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 11:53:40 +0100 Subject: [PATCH 06/11] Simplify create_venv(). See #463. --- scripts/init_venv.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index edb2f2afb..3cd49b8f5 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -158,11 +158,7 @@ def create_venv(): builder = venv.EnvBuilder(system_site_packages=system_site_packages, clear=clear, upgrade=upgrade, symlinks=symlinks, with_pip=True) - - context = builder.ensure_directories(g_path) - builder.create_configuration(context) - builder.setup_python(context) - builder.setup_scripts(context) + builder.create(g_path) def main(): From 907440d12c742a80f7e22f7801fd36645d1e6d77 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 11:56:44 +0100 Subject: [PATCH 07/11] Simplify option handling in create_venv(). See #463. --- scripts/init_venv.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index 3cd49b8f5..b7955784c 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -142,21 +142,14 @@ def link_pyqt(): def create_venv(): """Create a new venv.""" utils.print_title("Creating venv") - system_site_packages = True - - clear = False - if g_args.clear or g_args.force: - clear = True - - upgrade = False - if g_args.upgrade: - upgrade = True - - symlinks = False if os.name == 'nt' else True - + if os.name == 'nt': + symlinks = False + else: + symlinks = True + clear = g_args.clear or g_args.force builder = venv.EnvBuilder(system_site_packages=system_site_packages, - clear=clear, upgrade=upgrade, + clear=clear, upgrade=g_args.upgrade, symlinks=symlinks, with_pip=True) builder.create(g_path) From b8c54b5f025fba81aa4ce5f0fdac5f0f39c117f0 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 12:03:54 +0100 Subject: [PATCH 08/11] Don't use system_site_packages on non-Windows. See #463. --- scripts/init_venv.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index b7955784c..3ab0caaeb 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -142,11 +142,12 @@ def link_pyqt(): def create_venv(): """Create a new venv.""" utils.print_title("Creating venv") - system_site_packages = True if os.name == 'nt': symlinks = False + system_site_packages = True else: symlinks = True + system_site_packages = False clear = g_args.clear or g_args.force builder = venv.EnvBuilder(system_site_packages=system_site_packages, clear=clear, upgrade=g_args.upgrade, @@ -165,6 +166,9 @@ def main(): create_venv() + utils.print_title("Installing setuptools") + venv_python('-m', 'pip', 'install', 'setuptools') + utils.print_title("Calling: setup.py develop") venv_python('setup.py', 'develop') From b2646cb5c0c9c9690fd94123896d9603b148014a Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 13:23:33 +0100 Subject: [PATCH 09/11] Check if venv already exists. According to the documentation, Python should do that already: If the target directory already exists an error will be raised, unless the --clear or --upgrade option was provided. However that doesn't seem to be the case: http://bugs.python.org/issue23202 We do this by hand to make sure the user doesn't accidentally overwrite something. See #463. --- scripts/init_venv.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index 3ab0caaeb..e847741c7 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -164,6 +164,12 @@ def main(): sys.exit(1) g_path = os.path.abspath(g_args.path) + if os.path.exists(g_args.path) and not (g_args.force or g_args.clear or + g_args.upgrade): + print("{} does already exist! Use --clear or " + "--upgrade.".format(g_path), file=sys.stderr) + sys.exit(1) + create_venv() utils.print_title("Installing setuptools") From 6f1facac605016428e1f85f8b0c93a8f8891db98 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 13:47:27 +0100 Subject: [PATCH 10/11] Also get rid of system_site_packages on Windows. We can't symlink, so we copy the files instead. --- scripts/init_venv.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/init_venv.py b/scripts/init_venv.py index e847741c7..da4f459ca 100644 --- a/scripts/init_venv.py +++ b/scripts/init_venv.py @@ -109,10 +109,8 @@ def test_toolchain(): def link_pyqt(): """Symlink the systemwide PyQt/sip into the venv.""" - if os.name == 'nt': - return - utils.print_title("Softlinking PyQt5") - + action = "Copying" if os.name == 'nt' else "Softlinking" + utils.print_title("{} PyQt5".format(action)) sys_path = distutils.sysconfig.get_python_lib() venv_path = venv_python( '-c', 'from distutils.sysconfig import get_python_lib\n' @@ -130,13 +128,19 @@ def link_pyqt(): files += [os.path.basename(e) for e in globbed_sip] for fn in files: source = os.path.join(sys_path, fn) - link_name = os.path.join(venv_path, fn) + dest = os.path.join(venv_path, fn) if not os.path.exists(source): raise FileNotFoundError(source) - if os.path.exists(link_name): - os.unlink(link_name) - print('{} -> {}'.format(source, link_name)) - os.symlink(source, link_name) + if os.path.exists(dest): + os.unlink(dest) + print('{} -> {}'.format(source, dest)) + if os.name == 'nt': + if os.path.isdir(source): + shutil.copytree(source, dest) + else: + shutil.copy(source, dest) + else: + os.symlink(source, dest) def create_venv(): @@ -144,12 +148,10 @@ def create_venv(): utils.print_title("Creating venv") if os.name == 'nt': symlinks = False - system_site_packages = True else: symlinks = True - system_site_packages = False clear = g_args.clear or g_args.force - builder = venv.EnvBuilder(system_site_packages=system_site_packages, + builder = venv.EnvBuilder(system_site_packages=False, clear=clear, upgrade=g_args.upgrade, symlinks=symlinks, with_pip=True) builder.create(g_path) From d1d43b29dc56e9307ebcd8aa808cff83807ad0f1 Mon Sep 17 00:00:00 2001 From: Florian Bruhin Date: Fri, 23 Jan 2015 13:54:18 +0100 Subject: [PATCH 11/11] Remove virtualenv dependency in INSTALL.asciidoc. --- doc/INSTALL.asciidoc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/doc/INSTALL.asciidoc b/doc/INSTALL.asciidoc index f75b09e26..6616c50a2 100644 --- a/doc/INSTALL.asciidoc +++ b/doc/INSTALL.asciidoc @@ -13,7 +13,7 @@ qutebrowser should run on these systems: Install the dependencies via apt-get: ---- -# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit python-virtualenv +# apt-get install python3-pyqt5 python3-pyqt5.qtwebkit ---- To generate the documentation for the `:help` command, when using the git @@ -25,14 +25,14 @@ repository (rather than a release): ---- Then run the supplied script to run qutebrowser inside a -http://virtualenv.readthedocs.org/en/latest/[virtualenv]: +https://docs.python.org/3/library/venv.html[virtual environment]: ---- # python3 scripts/init_venv.py ---- This installs all needed Python dependencies in a `.venv` subfolder. The -system-wide Qt5/PyQt5 installations are symlinked into the virtualenv. +system-wide Qt5/PyQt5 installations are symlinked into the virtual environment. You can then create a simple wrapper script to start qutebrowser somewhere in your `$PATH` (e.g. `/usr/local/bin/qutebrowser` or `~/bin/qutebrowser`): @@ -102,19 +102,16 @@ Python 3 (be sure to install pip). * Use the installer from http://www.riverbankcomputing.com/software/pyqt/download5[Riverbank computing] to get Qt and PyQt5. -* Run `pip install virtualenv` or -http://www.lfd.uci.edu/~gohlke/pythonlibs/#virtualenv[the installer from here] -to install virtualenv. Then run the supplied script to run qutebrowser inside a -http://virtualenv.readthedocs.org/en/latest/[virtualenv]: +https://docs.python.org/3/library/venv.html[virtual environment]: ---- # python3 scripts/init_venv.py ---- This installs all needed Python dependencies in a `.venv` subfolder. The -system-wide Qt5/PyQt5 installations are used in the virtualenv. +system-wide Qt5/PyQt5 installations are used in the virtual environment. On OS X -------