deployment.py
Go to the documentation of this file.
1 # -*- coding: utf-8 -*-
2 # Copyright (c) 2013 - 2014 Spotify AB
3 
4 # This file is part of dh-virtualenv.
5 
6 # dh-virtualenv is free software: you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License as
8 # published by the Free Software Foundation, either version 2 of the
9 # License, or (at your option) any later version.
10 
11 # dh-virtualenv is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
15 
16 # You should have received a copy of the GNU General Public License
17 # along with dh-virtualenv. If not, see
18 # <http://www.gnu.org/licenses/>.
19 
20 # Note for catkin_virtualenv users: local changes from upstream are marked with (pbovbel)
21 from __future__ import print_function
22 
23 
24 import os
25 import re
26 import shutil
27 import subprocess
28 import tempfile
29 
30 ROOT_ENV_KEY = 'DH_VIRTUALENV_INSTALL_ROOT'
31 DEFAULT_INSTALL_DIR = '/opt/venvs/'
32 PYTHON_INTERPRETERS = ['python', 'pypy', 'ipy', 'jython']
33 _PYTHON_INTERPRETERS_REGEX = r'\(' + r'\|'.join(PYTHON_INTERPRETERS) + r'\)'
34 
35 
36 # (pbovbel) Log subprocess calls
37 def check_call(cmd, *args, **kwargs):
38  print(' '.join(cmd))
39  return subprocess.check_call(cmd, *args, **kwargs)
40 
41 
42 class Deployment(object):
43  def __init__(self,
44  package,
45  extra_urls=[],
46  preinstall=[],
47  pip_tool='pip',
48  upgrade_pip=False,
49  index_url=None,
50  setuptools=False,
51  python=None,
52  builtin_venv=False,
53  builtin_pip=False,
54  sourcedirectory=None,
55  verbose=False,
56  extra_pip_arg=[],
57  extra_virtualenv_arg=[],
58  use_system_packages=False,
59  skip_install=False,
60  install_suffix=None,
61  log_file=tempfile.NamedTemporaryFile().name, # (pbovbel): addition
62  pip_version=None, # (pbovbel): addition
63  requirements_filename='requirements.txt'):
64 
65  self.package = package
66 
67  install_root = os.environ.get(ROOT_ENV_KEY, DEFAULT_INSTALL_DIR)
68  self.install_suffix = install_suffix
69 
70  # (pbovbel): override the debian_root directory to allow a custom install path
71  # self.debian_root = os.path.join(
72  # 'debian', package, install_root.lstrip('/'))
73  self.debian_root = '.'
74 
75  if install_suffix is None:
76  self.virtualenv_install_dir = os.path.join(install_root, self.package)
77  self.package_dir = os.path.join(self.debian_root, package)
78  else:
79  self.virtualenv_install_dir = os.path.join(install_root, install_suffix)
80  self.package_dir = os.path.join(self.debian_root, install_suffix)
81 
82  self.bin_dir = os.path.join(self.package_dir, 'bin')
83  self.local_bin_dir = os.path.join(self.package_dir, 'local', 'bin')
84 
85  self.preinstall = preinstall
86  self.upgrade_pip = upgrade_pip
87  self.pip_version = pip_version
88  self.extra_virtualenv_arg = extra_virtualenv_arg
89  self.verbose = verbose
90  self.setuptools = setuptools
91  self.python = python
92  self.builtin_venv = builtin_venv
93  self.sourcedirectory = '.' if sourcedirectory is None else sourcedirectory
94  self.use_system_packages = use_system_packages
95  self.skip_install = skip_install
96  self.requirements_filename = requirements_filename
97 
98  # We need to prefix the pip run with the location of python
99  # executable. Otherwise it would just blow up due to too long
100  # shebang-line.
101  python = self.venv_bin('python')
102  if builtin_pip:
103  self.pip_preinstall_prefix = [python, '-m', 'pip']
104  self.pip_prefix = [python, '-m', pip_tool]
105  else:
106  self.pip_preinstall_prefix = [python, self.venv_bin('pip')]
107  self.pip_prefix = [python, self.venv_bin(pip_tool)]
108  self.pip_args = ['install']
109 
110  if self.verbose:
111  self.pip_args.append('-v')
112 
113  if index_url:
114  self.pip_args.append('--index-url={0}'.format(index_url))
115  self.pip_args.extend([
116  '--extra-index-url={0}'.format(url) for url in extra_urls
117  ])
118 
119  # (pbovbel): make logging optional, since --log overrides -q flag for pip
120  self.log_file = log_file
121  if self.log_file:
122  self.pip_args.append('--log={0}'.format(os.path.abspath(self.log_file)))
123 
124  # Keep a copy with well-suported options only (for upgrading pip itself)
125  self.pip_upgrade_args = self.pip_args[:]
126 
127  # Add in any user supplied pip args
128  self.pip_args.extend(extra_pip_arg)
129 
130  # (pbovbel) Set pip_upgrade_args here to keep flags like -q, disregard L111 above.
131  self.pip_upgrade_args = self.pip_args
132 
133  @classmethod
134  def from_options(cls, package, options):
135  verbose = options.verbose or os.environ.get('DH_VERBOSE') == '1'
136  return cls(package,
137  extra_urls=options.extra_index_url,
138  preinstall=options.preinstall,
139  pip_tool=options.pip_tool,
140  upgrade_pip=options.upgrade_pip,
141  index_url=options.index_url,
142  setuptools=options.setuptools,
143  python=options.python,
144  builtin_venv=options.builtin_venv,
145  builtin_pip=options.builtin_pip,
146  sourcedirectory=options.sourcedirectory,
147  verbose=verbose,
148  extra_pip_arg=options.extra_pip_arg,
149  extra_virtualenv_arg=options.extra_virtualenv_arg,
150  use_system_packages=options.use_system_packages,
151  skip_install=options.skip_install,
152  install_suffix=options.install_suffix,
153  requirements_filename=options.requirements_filename)
154 
155  def clean(self):
156  shutil.rmtree(self.debian_root)
157 
158  def create_virtualenv(self):
159  if self.builtin_venv:
160  virtualenv = [self.python, '-m', 'venv']
161 
162  if self.use_system_packages:
163  virtualenv.append('--system-site-packages')
164 
165  else:
166  virtualenv = ['virtualenv']
167 
168  if self.python:
169  virtualenv.extend(('--python', self.python))
170 
171  if self.use_system_packages:
172  virtualenv.append('--system-site-packages')
173  else:
174  virtualenv.append('--no-site-packages')
175 
176  if self.setuptools:
177  virtualenv.append('--setuptools')
178 
179  if self.verbose:
180  virtualenv.append('--verbose')
181 
182  # Add in any user supplied pip args
183  if self.extra_virtualenv_arg:
184  virtualenv.extend(self.extra_virtualenv_arg)
185 
186  virtualenv.append(self.package_dir)
187  check_call(virtualenv)
188 
189  def venv_bin(self, binary_name):
190  return os.path.abspath(os.path.join(self.bin_dir, binary_name))
191 
192  def pip_preinstall(self, *args):
193  return self.pip_preinstall_prefix + self.pip_args + list(args)
194 
195  def pip(self, *args):
196  return self.pip_prefix + self.pip_args + list(args)
197 
199  # Install preinstall stage packages. This is handy if you need
200  # a custom package to install dependencies (think something
201  # along lines of setuptools), but that does not get installed
202  # by default virtualenv.
203  if self.upgrade_pip:
204  # (pbovbel) allow pinning the pip version
205  pip_package = 'pip==' + self.pip_version if self.pip_version else 'pip'
206  # First, bootstrap pip with a reduced option set (well-supported options)
207  print(self.pip_preinstall_prefix + self.pip_upgrade_args + ['-U', pip_package])
208  check_call(self.pip_preinstall_prefix + self.pip_upgrade_args + ['-U', pip_package])
209  if self.preinstall:
211 
212  requirements_path = os.path.join(self.sourcedirectory, self.requirements_filename)
213  if os.path.exists(requirements_path):
214  check_call(self.pip('-r', requirements_path))
215 
216  def run_tests(self):
217  python = self.venv_bin('python')
218  setup_py = os.path.join(self.sourcedirectory, 'setup.py')
219  if os.path.exists(setup_py):
220  check_call([python, 'setup.py', 'test'], cwd=self.sourcedirectory)
221 
222  def find_script_files(self):
223  """Find list of files containing python shebangs in the bin directory"""
224  command = ['grep', '-l', '-r',
225  '-e', r'^#!.*bin/\(env \)\?{0}'.format(_PYTHON_INTERPRETERS_REGEX),
226  '-e', r"^'''exec.*bin/{0}".format(_PYTHON_INTERPRETERS_REGEX),
227  self.bin_dir]
228  grep_proc = subprocess.Popen(command, stdout=subprocess.PIPE)
229  files, stderr = grep_proc.communicate()
230  return set(f for f in files.decode('utf-8').strip().split('\n') if f)
231 
232  def fix_shebangs(self):
233  """Translate /usr/bin/python and /usr/bin/env python shebang
234  lines to point to our virtualenv python.
235  """
236  pythonpath = os.path.join(self.virtualenv_install_dir, 'bin/python')
237  for f in self.find_script_files():
238  regex = (
239  r's-^#!.*bin/\(env \)\?{names}\"\?-#!{pythonpath}-;'
240  r"s-^'''exec'.*bin/{names}-'''exec' {pythonpath}-"
241  ).format(names=_PYTHON_INTERPRETERS_REGEX, pythonpath=re.escape(pythonpath))
242  check_call(['sed', '-i', regex, f])
243 
244  def fix_activate_path(self):
245  """Replace the `VIRTUAL_ENV` path in bin/activate to reflect the
246  post-install path of the virtualenv.
247  """
248  activate_settings = [
249  [
250  'VIRTUAL_ENV="{0}"'.format(self.virtualenv_install_dir),
251  r'^VIRTUAL_ENV=.*$',
252  "activate"
253  ],
254  [
255  'setenv VIRTUAL_ENV "{0}"'.format(self.virtualenv_install_dir),
256  r'^setenv VIRTUAL_ENV.*$',
257  "activate.csh"
258  ],
259  [
260  'set -gx VIRTUAL_ENV "{0}"'.format(self.virtualenv_install_dir),
261  r'^set -gx VIRTUAL_ENV.*$',
262  "activate.fish"
263  ],
264  ]
265 
266  for activate_args in activate_settings:
267  virtualenv_path = activate_args[0]
268  pattern = re.compile(activate_args[1], flags=re.M)
269  activate_file = activate_args[2]
270 
271  with open(self.venv_bin(activate_file), 'r+') as fh:
272  content = pattern.sub(virtualenv_path, fh.read())
273  fh.seek(0)
274  fh.truncate()
275  fh.write(content)
276 
277  def install_package(self):
278  if not self.skip_install:
279  check_call(self.pip('.'), cwd=os.path.abspath(self.sourcedirectory))
280 
282  # The virtualenv might end up with a local folder that points outside the package
283  # Specifically it might point at the build environment that created it!
284  # Make those links relative
285  # See https://github.com/pypa/virtualenv/commit/5cb7cd652953441a6696c15bdac3c4f9746dfaa1
286  local_dir = os.path.join(self.package_dir, "local")
287  if not os.path.isdir(local_dir):
288  return
289  elif os.path.samefile(self.package_dir, local_dir):
290  # "local" points directly to its containing directory
291  os.unlink(local_dir)
292  os.symlink(".", local_dir)
293  return
294 
295  for d in os.listdir(local_dir):
296  path = os.path.join(local_dir, d)
297  if not os.path.islink(path):
298  continue
299 
300  existing_target = os.readlink(path)
301  if not os.path.isabs(existing_target):
302  # If the symlink is already relative, we don't
303  # want to touch it.
304  continue
305 
306  new_target = os.path.relpath(existing_target, local_dir)
307  os.unlink(path)
308  os.symlink(new_target, path)
def from_options(cls, package, options)
Definition: deployment.py:134
def venv_bin(self, binary_name)
Definition: deployment.py:189
def check_call(cmd, args, kwargs)
Definition: deployment.py:37
def __init__(self, package, extra_urls=[], preinstall=[], pip_tool='pip', upgrade_pip=False, index_url=None, setuptools=False, python=None, builtin_venv=False, builtin_pip=False, sourcedirectory=None, verbose=False, extra_pip_arg=[], extra_virtualenv_arg=[], use_system_packages=False, skip_install=False, install_suffix=None, log_file=tempfile.NamedTemporaryFile().name, pip_version=None, requirements_filename='requirements.txt')
Definition: deployment.py:63


catkin_virtualenv
Author(s): Paul Bovbel
autogenerated on Mon Jul 1 2019 19:33:08