Source Code for Module rosmake.engine

  1  #! /usr/bin/env python 
  2   
  3  # Copyright (c) 2009, Willow Garage, Inc. 
  4  # All rights reserved. 
  5  #  
  6  # Redistribution and use in source and binary forms, with or without 
  7  # modification, are permitted provided that the following conditions are met: 
  8  #  
  9  #     * Redistributions of source code must retain the above copyright 
 10  #       notice, this list of conditions and the following disclaimer. 
 11  #     * Redistributions in binary form must reproduce the above copyright 
 12  #       notice, this list of conditions and the following disclaimer in the 
 13  #       documentation and/or other materials provided with the distribution. 
 14  #     * Neither the name of Willow Garage, Inc. nor the names of its 
 15  #       contributors may be used to endorse or promote products derived from 
 16  #       this software without specific prior written permission. 
 17  #  
 18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
 19  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
 20  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 21  # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
 22  # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 23  # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 24  # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 25  # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 26  # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 27  # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 28  # POSSIBILITY OF SUCH DAMAGE. 
 29   
 30  # Author Tully Foote/tfoote@willowgarage.com 
 31   
 32  from __future__ import print_function 
 33   
 34  import os 
 35  import re 
 36  import signal 
 37  import sys 
 38  import subprocess 
 39  import time 
 40  import threading 
 41  import traceback 
 42   
 43  import rospkg 
 44   
 45  try:  
 46      from exceptions import SystemExit #Python 2.x 
 47  except ImportError:  
 48      pass #Python 3.x (in Python 3, 'exceptions' is always imported)  
 49   
 50  from operator import itemgetter 
 51   
 52  from . import parallel_build 
 53  from . import package_stats 
 54   
 55  from optparse import OptionParser 
 56   
 57  # #3883 
 58  _popen_lock = threading.Lock() 
 59   
 60 -def make_command(): 
 61      """ 
 62      @return: name of 'make' command 
 63      @rtype: str 
 64      """ 
 65      return os.environ.get("MAKE", "make") 
 66   
 67  # this is a copy of the roslogging utility. it's been moved here as it is a common 
 68  # routine for programs using accessing ROS directories 
 69 -def makedirs_with_parent_perms(p): 
 70      """ 
 71      Create the directory using the permissions of the nearest 
 72      (existing) parent directory. This is useful for logging, where a 
 73      root process sometimes has to log in the user's space. 
 74      @param p: directory to create 
 75      @type  p: str 
 76      """     
 77      p = os.path.abspath(p) 
 78      parent = os.path.dirname(p) 
 79      # recurse upwards, checking to make sure we haven't reached the 
 80      # top 
 81      if not os.path.exists(p) and p and parent != p: 
 82          makedirs_with_parent_perms(parent) 
 83          s = os.stat(parent) 
 84          os.mkdir(p) 
 85   
 86          # if perms of new dir don't match, set anew 
 87          s2 = os.stat(p) 
 88          if s.st_uid != s2.st_uid or s.st_gid != s2.st_gid: 
 89              os.chown(p, s.st_uid, s.st_gid) 
 90          if s.st_mode != s2.st_mode: 
 91              os.chmod(p, s.st_mode)     
 92   
 93 -class Printer: 
 94     # storage for the instance reference 
 95      __instance = None 
 96   
 97 -    def __init__(self): 
 98          """ Create singleton instance """ 
 99          # Check whether we already have an instance 
100          if Printer.__instance is None: 
101              # Create and remember instance 
102              Printer.__instance = Printer.__impl() 
103   
104          # Store instance reference as the only member in the handle 
105          self.__dict__['_Printer__instance'] = Printer.__instance 
106   
107 -    def __getattr__(self, attr): 
108          """ Delegate access to implementation """ 
109          return getattr(self.__instance, attr) 
110   
111 -    def __setattr__(self, attr, value): 
112          """ Delegate access to implementation """ 
113          return setattr(self.__instance, attr, value) 
114       
115 -    def __enter__(self): 
116          """Pass through for the __enter__ function for the __instance""" 
117          return self.__instance.__enter__() 
118       
119 -    def __exit__(self, mtype, value, tb): 
120          """Pass through for the __exit__ function for the __instance""" 
121          return self.__instance.__exit__(mtype, value, tb) 
122       
123 -    class __impl(threading.Thread): 
124 -        def __init__(self): 
125              threading.Thread.__init__(self) 
126              self.build_queue = None 
127              self.condition = threading.Condition() 
128              self.running = True 
129              self.done = False 
130              self.status = "" 
131              self.verbose = False 
132              self.full_verbose = False 
133              self.duration = 1./10. 
134              self._last_status = None 
135   
136              # Rosmake specific data 
137              self.cache_argument = None 
138              self.cache_right = '' 
139              self.pkg_start_times = {} 
140   
141 -        def shutdown(self): 
142              self.running = False 
143              cycles = 10 
144              for i in range(0,cycles):# sleep for at least 2 cycles of the status testing 'cycles' times 
145                  if self.done: 
146                      #print "SUCCESSFULLY SHUTDOWN" 
147                      return True 
148                  #print "Sleeping for %f FOR SHUTDOWN.  %d threads running"%(max(self.duration/cycles*2, 0.01), threading.activeCount()) 
149                  time.sleep(max(self.duration, 0.1)/cycles*2)  
150              raise Exception("Failed to shutdown status thread in %.2f seconds"%(self.duration * 2)) 
151   
152 -        def __enter__(self): 
153              self.start() 
154 -        def __exit__(self, mtype, value, tb): 
155              self.shutdown() 
156              if value: 
157                  if not mtype == type(SystemExit()): 
158                      traceback.print_exception(mtype, value, tb) 
159                  else: 
160                      sys.exit(value) 
161   
162 -        def run(self): 
163              while self.running: 
164                  #shutdown if duration set to zero 
165                  if self.duration <= 0: 
166                      self.running = False 
167                      break 
168                  self.set_status_from_cache() 
169                  if len(self.pkg_start_times.keys()) > 0: 
170                      n = self.terminal_width() - len(self.status) 
171                      status = self.status 
172                      if n > 0: 
173                          status = " "*n + self.status 
174                      if status != self._last_status: 
175                          self._print_status("%s"%status) 
176                          self._last_status = status 
177                  time.sleep(self.duration)  
178              self.done = True 
179              #print "STATUS THREAD FINISHED" 
180   
181 -        def rosmake_cache_info(self, argument, start_times, right): 
182              self.cache_argument = argument 
183              self.pkg_start_times = start_times 
184              self.cache_right = right 
185   
186 -        def rosmake_pkg_times_to_string(self, start_times): 
187              threads = [] 
188              for p, t in sorted(start_times.items(), key=itemgetter(1)): #py3k 
189                  threads.append("[ %s: %.1f sec ]"%(p, time.time() - t)) 
190   
191              return " ".join(threads) 
192   
193 -        def set_status_from_cache(self): 
194              if self.cache_argument: 
195                  self.set_status("[ make %s ] "%self.cache_argument + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
196              else: 
197                  self.set_status("[ make ] " + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
198   
199 -        def set_status(self, left, right = ''): 
200              header = "[ rosmake ] " 
201              h = len(header) 
202              l = len(left) 
203              r = len(right) 
204              w = self.terminal_width() 
205              if l + r < w - h: 
206                  padding = w - h - l - r 
207                  self.status = header + left + " "*padding + right 
208              else: 
209                  self.status = header + left[:(w - h - r - 4)] + "... " + right 
210   
211 -        def print_all(self, s, thread_name=None): 
212              if thread_name is None: 
213                  str = "[ rosmake ] %s"%s 
214                   
215   
216              else: 
217                  str = "[rosmake-%s] %s"%(thread_name, s) 
218              sys.stdout.write(self.pad_str_to_width(str, self.terminal_width())+"\n") 
219              sys.stdout.flush() 
220   
221 -        def print_verbose(self, s, thread_name=None): 
222              if self.verbose or self.full_verbose: 
223                  self.print_all(s, thread_name=thread_name) 
224   
225 -        def print_full_verbose(self, s): 
226              if self.full_verbose: 
227                  print("[ rosmake ] %s"%(s)) 
228   
229 -        def print_tail(self, s, tail_lines=40): 
230              lines = s.splitlines() 
231              if self.full_verbose: 
232                  tail_lines = len(lines) 
233   
234              num_lines = min(len(lines), tail_lines) 
235              if num_lines == tail_lines: 
236                  print("[ rosmake ] Last %d lines"%(num_lines)) 
237              else: 
238                  print("[ rosmake ] All %d lines"%(num_lines)) 
239              print("{" + "-"*79) 
240              for l in range(-num_lines, -1): 
241                  print("  %s"%(lines[l])) 
242              print("-"*79 + "}") 
243   
244 -        def _print_status(self, s): 
245              sys.stdout.write("%s\r"%(s)) 
246              sys.stdout.flush() 
247   
248          @staticmethod 
249 -        def terminal_width(): 
250              """Estimate the width of the terminal""" 
251              width = 0 
252              try: 
253                  import struct, fcntl, termios 
254                  s = struct.pack('HHHH', 0, 0, 0, 0) 
255                  x = fcntl.ioctl(1, termios.TIOCGWINSZ, s) 
256                  width = struct.unpack('HHHH', x)[1] 
257              except IOError: 
258                  pass 
259              if width <= 0: 
260                  try: 
261                      width = int(os.environ['COLUMNS']) 
262                  except: 
263                      pass 
264              if width <= 0: 
265                  width = 80 
266   
267              return width 
268   
269          @staticmethod 
270 -        def pad_str_to_width(str, width): 
271              """ Pad the string to be terminal width""" 
272              length = len(str) 
273              excess = 0 
274              if length < width: 
275                  excess = width - length 
276              return str + " "* excess 
277   
278   
279   
280 -class RosMakeAll: 
281 -    def __init__(self): 
282          self._result_lock = threading.Lock() 
283   
284          self.rospack = rospkg.RosPack() 
285          self.rosstack = rospkg.RosStack() 
286   
287          self.printer = Printer() 
288          self.result = {} 
289          self.paths = {} 
290          self.dependency_tracker = parallel_build.DependencyTracker(rospack=self.rospack) 
291          self.flag_tracker = package_stats.PackageFlagTracker(self.dependency_tracker) 
292          self.output = {} 
293          self.profile = {} 
294          self.ros_parallel_jobs = 0 
295          self.build_list = [] 
296          self.start_time = time.time() 
297          self.log_dir = "" 
298          self.logging_enabled = True 
299   
300 -    def num_packages_built(self): 
301          """ 
302          @return: number of packages that were built 
303          @rtype: int 
304          """ 
305          return len(list(self.result[argument].keys())) #py3k 
306   
307 -    def update_status(self, argument, start_times, right): 
308          self.printer.rosmake_cache_info(argument, start_times, right) 
309   
310 -    def build_or_recurse(self,p): 
311          if p in self.build_list: 
312              return 
313          for d in self.dependency_tracker.get_deps_1(p): 
314              self.build_or_recurse(d) 
315          try: # append it ot the list only if present 
316            self.rospack.get_path(p) 
317            self.build_list.append(p) 
318          except rospkg.ResourceNotFound as ex: 
319            if not self.robust_build: 
320              self.printer.print_all("Exiting due to missing package: %s"%ex) 
321              sys.exit(-1) 
322            else: 
323              self.printer.print_all("!"*20 + " Package %s does not exist. %s"%(p, ex) + "!"*20) 
324   
325   
326 -    def parallel_build_pkgs(self, build_queue, argument = None, threads = 1): 
327          self.profile[argument] = {} 
328          self.output[argument] = {} 
329          with self._result_lock: 
330              if argument not in self.result.keys(): 
331                  self.result[argument] = {} 
332   
333          cts = [] 
334          for i in range(0, threads): 
335            ct = parallel_build.CompileThread(str(i), build_queue, self, argument) 
336            #print "TTTH starting thread ", ct 
337            ct.start() 
338            cts.append(ct) 
339          for ct in cts: 
340            try: 
341              #print "TTTT Joining", ct 
342              ct.join() 
343              #print "TTTH naturally ended thread", ct 
344            except KeyboardInterrupt: 
345              self.printer.print_all( "TTTH Caught KeyboardInterrupt. Stopping build.") 
346              build_queue.stop() 
347              ct.join() 
348            except: #catch all 
349                self.printer.print_all("TTTH OTHER exception thrown!!!!!!!!!!!!!!!!!!!!!") 
350                ct.join() 
351          #print "All threads joined" 
352          all_pkgs_passed = True 
353          with self._result_lock: 
354              for v in self.result[argument].values(): 
355                  all_pkgs_passed = v and all_pkgs_passed 
356   
357          build_passed = build_queue.succeeded() and all_pkgs_passed 
358          return build_passed 
359   
360      # This function taken from 
361      # http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html 
362 -    def _subprocess_setup(self): 
363          # Python installs a SIGPIPE handler by default. This is usually not 
364          # what non-Python subprocesses expect. 
365          signal.signal(signal.SIGPIPE, signal.SIG_DFL) 
366   
367 -    def _build_package(self, package, argument=None): 
368          """ 
369          Lower-level routine for building a package. Handles execution of actual build command. 
370          @param package: package name 
371          @type  package: str 
372          """ 
373          local_env = os.environ.copy() 
374          if self.ros_parallel_jobs > 0: 
375              local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (self.ros_parallel_jobs, self.ros_parallel_jobs) 
376          elif "ROS_PARALLEL_JOBS" not in os.environ: #if no environment setup and no args fall back to # cpus 
377              # num_cpus check can (on OS X) trigger a Popen(), which has 
378              #the multithreading bug we wish to avoid on Py2.7. 
379              with _popen_lock: 
380                  num_cpus = parallel_build.num_cpus() 
381                  local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (num_cpus, num_cpus) 
382          local_env['SVN_CMDLINE'] = "svn --non-interactive" 
383          cmd = ["bash", "-c", "cd %s && %s "%(self.rospack.get_path(package), make_command()) ] #UNIXONLY 
384          if argument: 
385              cmd[-1] += argument 
386          self.printer.print_full_verbose (cmd) 
387          # #3883: make sure only one Popen command occurs at a time due to 
388          # http://bugs.python.org/issue13817 
389          with _popen_lock: 
390              command_line = subprocess.Popen(cmd, stdout=subprocess.PIPE,  stderr=subprocess.STDOUT, env=local_env, preexec_fn=self._subprocess_setup) 
391          (pstd_out, pstd_err) = command_line.communicate() # pstd_err should be None due to pipe above 
392          return (command_line.returncode, pstd_out) 
393   
394 -    def build(self, p, argument = None, robust_build=False): 
395          """ 
396          Build package 
397          @param p: package name 
398          @type  p: str 
399          """ 
400          return_string = "" 
401          try: 
402              # warn if ROS_BUILD_BLACKLIST encountered if applicable 
403              # do not build packages for which the build has failed 
404              if argument == "test":  # Tests are not build dependent 
405                  failed_packages = [] 
406              else: 
407                  with self._result_lock: 
408                      failed_packages = [j for j in self.result[argument] if not self.result[argument][j] == True] 
409   
410              (buildable, error, why) = self.flag_tracker.can_build(p, self.skip_blacklist, failed_packages) 
411              if buildable or self.robust_build: 
412                  start_time = time.time() 
413                  (returncode, pstd_out) = self._build_package(p, argument) 
414                  self.profile[argument][p] = time.time() - start_time 
415                  self.output[argument][p] = pstd_out 
416                  if argument: 
417                      log_type = "build_%s"%argument 
418                  else: 
419                      log_type = "build" 
420                  if not returncode: 
421                      self.printer.print_full_verbose( pstd_out) 
422                      with self._result_lock: 
423                          self.result[argument][p] = True 
424   
425                      num_warnings = len(re.findall("warning:", pstd_out)) 
426                      if num_warnings > 0: 
427                          return_string =  ("[PASS] [ %.2f seconds ] -- WARNING: %d compiler warnings"%(self.profile[argument][p], num_warnings)) 
428                      else: 
429                          return_string =  ("[PASS] [ %.2f seconds ]"%( self.profile[argument][p])) 
430                      self.output_to_file(p, log_type, pstd_out, num_warnings > 0) 
431                  else: 
432                      success = False 
433                      no_target = len(re.findall("No rule to make target", pstd_out)) > 0 
434                      interrupt = len(re.findall("Interrupt", pstd_out)) > 0 
435                      if no_target: 
436                          return_string = ( "[SKIP] No rule to make target %s"%( argument)) 
437                          success = True 
438                      elif interrupt: 
439                          return_string = ("[Interrupted]" ) 
440                      else: 
441                          return_string = ( "[FAIL] [ %.2f seconds ]"%( self.profile[argument][p])) 
442                      with self._result_lock: 
443                          self.result[argument][p] = True if no_target else False 
444   
445                      if success == False: #don't print tail if [SKIP] target 
446                          self.printer.print_tail( pstd_out) 
447                      self.output_to_file(p, log_type, pstd_out, always_print= not (no_target or interrupt)) 
448   
449                      return (success, return_string) 
450              else: 
451                  with self._result_lock: 
452                      self.result[argument][p] = error 
453   
454                  return_string += why 
455                  return(error, return_string) 
456              return (True, return_string) # this means that we didn't error in any case above 
457          except rospkg.ResourceNotFound as ex: 
458              with self._result_lock: 
459                  self.result[argument][p] = False 
460              self.printer.print_verbose ("[SKIP] Package %s not found\n" % p) 
461              self.output[argument][p] = "Package not found %s"%ex 
462              return (False, return_string) 
463               
464   
465 -    def output_to_file(self, package, log_type, stdout, always_print= False): 
466          if not self.logging_enabled: 
467              return 
468          package_log_dir = os.path.join(self.log_dir, package) 
469   
470          std_out_filename = os.path.join(package_log_dir, log_type + "_output.log") 
471          if not os.path.exists (package_log_dir): 
472              makedirs_with_parent_perms(package_log_dir) 
473          with open(std_out_filename, 'w') as stdout_file: 
474              stdout_file.write(stdout) 
475              print_string = "Output from build of package %s written to:\n[ rosmake ]    %s"%(package, std_out_filename) 
476              if always_print: 
477                  self.printer.print_all(print_string) 
478              else: 
479                  self.printer.print_full_verbose(print_string) 
480   
481 -    def generate_summary_output(self, log_dir): 
482          if not self.logging_enabled: 
483              return 
484   
485          self.printer.print_all("Results:") 
486          if 'clean' in self.result.keys(): 
487              self.printer.print_all("Cleaned %d packages."%len(self.result['clean'])) 
488          if None in self.result.keys(): 
489              build_failure_count = len([p for p in self.result[None].keys() if self.result[None][p] == False]) 
490              self.printer.print_all("Built %d packages with %d failures."%(len(self.result[None]), build_failure_count)) 
491          if 'test' in self.result.keys(): 
492              test_failure_count = len([p for p in self.result['test'].keys() if self.result['test'][p] == False]) 
493              self.printer.print_all("Tested %d packages with %d failures."%(len(self.result['test']), test_failure_count)) 
494          self.printer.print_all("Summary output to directory") 
495          self.printer.print_all("%s"%self.log_dir) 
496          if self.rejected_packages: 
497              self.printer.print_all("WARNING: Skipped command line arguments: %s because they could not be resolved to a stack name or a package name. "%self.rejected_packages) 
498   
499                              
500   
501          if None in self.result.keys(): 
502              if len(self.result[None].keys()) > 0: 
503                  buildfail_filename = os.path.join(log_dir, "buildfailures.txt") 
504                  with open(buildfail_filename, 'w') as bf: 
505                      bf.write("Build failures:\n") 
506                      for key in self.build_list: 
507                          if key in self.result[None].keys() and self.result[None][key] == False: 
508                              bf.write("%s\n"%key) 
509                  if  None in self.output.keys(): 
510                      buildfail_context_filename = os.path.join(log_dir, "buildfailures-with-context.txt") 
511                      with open(buildfail_context_filename, 'w') as bfwc: 
512                          bfwc.write("Build failures with context:\n") 
513                          for key in self.build_list: 
514                              if key in self.result[None].keys() and self.result[None][key] == False: 
515                                  bfwc.write("---------------------\n") 
516                                  bfwc.write("%s\n"%key) 
517                                  if key in self.output[None]: 
518                                      bfwc.write(self.output[None][key]) 
519   
520          if "test" in self.result.keys(): 
521              if len(self.result["test"].keys()) > 0: 
522                  testfail_filename = os.path.join(log_dir, "testfailures.txt") 
523                  with open(testfail_filename, 'w') as btwc: 
524                      btwc.write("Test failures:\n") 
525                      for key in self.build_list: 
526                          if key in self.result["test"].keys() and self.result["test"][key] == False: 
527                              btwc.write("%s\n"%key) 
528   
529                  if "test" in self.output.keys(): 
530                      testfail_filename = os.path.join(log_dir, "testfailures-with-context.txt") 
531                      with open(testfail_filename, 'w') as btwc: 
532                          btwc.write("Test failures with context:\n") 
533                          for key in self.build_list: 
534                              if key in self.result["test"].keys() and self.result["test"][key] == False: 
535                                  btwc.write("%s\n"%key) 
536                                  if key in self.output["test"]: 
537                                      btwc.write(self.output["test"][key]) 
538   
539          profile_filename = os.path.join(log_dir, "profile.txt") 
540          with open(profile_filename, 'w') as pf: 
541              pf.write(self.get_profile_string()) 
542                               
543                               
544   
545 -    def get_profile_string(self): 
546          output = '--------------\nProfile\n--------------\n' 
547          total = 0.0 
548          count = 1 
549          for key in self.build_list: 
550              build_results = ["[Not Built ]", "[  Built   ]", "[Build Fail]"]; 
551              test_results =  ["[Untested ]", "[Test Pass]", "[Test Fail]"]; 
552              build_result = 0 
553              test_result = 0 
554              test_time = 0.0 
555              build_time = 0.0 
556   
557              if None in self.result.keys(): 
558                  if key in self.result[None].keys(): 
559                      if self.result[None][key] == True: 
560                          build_result = 1 
561                      else: 
562                          build_result = 2 
563   
564              if "test" in self.profile.keys(): 
565                  if key in self.result["test"].keys(): 
566                      if self.result["test"][key] == True: 
567                          test_result = 1 
568                      else: 
569                          test_result = 2 
570   
571              if None in self.profile.keys(): 
572                  if key in self.profile[None].keys(): 
573                      build_time = self.profile[None][key] 
574   
575              if "test" in self.profile.keys(): 
576                  if key in self.profile["test"].keys(): 
577                      test_time = self.profile["test"][key] 
578                   
579                       
580              output = output + "%3d: %s in %.2f %s in %.2f --- %s\n"% (count, build_results[build_result], build_time , test_results[test_result], test_time, key) 
581              total = total + build_time 
582              count = count + 1 
583   
584          elapsed_time  =  self.finish_time - self.start_time 
585          output = output + "----------------\n" + "%.2f Cumulative,  %.2f Elapsed, %.2f Speedup \n"%(total, elapsed_time, float(total) / float(elapsed_time)) 
586          return output 
587   
588 -    def main(self): 
589          """ 
590          main command-line entrypoint 
591          """ 
592          parser = OptionParser(usage="usage: %prog [options] [PACKAGE]...", 
593                                description="rosmake recursively builds all dependencies before building a package", prog='rosmake') 
594          parser.add_option("--test-only", dest="test_only", default=False, 
595                            action="store_true", help="only run tests") 
596          parser.add_option("-t", dest="test", default=False, 
597                            action="store_true", help="build and test packages") 
598          parser.add_option("-a", "--all", dest="build_all", default=False, 
599                            action="store_true", help="select all packages") 
600          parser.add_option("-i", "--mark-installed", dest="mark_installed", default=False, 
601                            action="store_true", help="On successful build, mark specified packages as installed with ROS_NOBUILD") 
602          parser.add_option("-u", "--unmark-installed", dest="unmark_installed", default=False, 
603                            action="store_true", help="Remove ROS_NOBUILD from the specified packages.  This will not build anything.") 
604          parser.add_option("-v", dest="verbose", default=False, 
605                            action="store_true", help="display errored builds") 
606          parser.add_option("-r","-k", "--robust", dest="best_effort", default=False, 
607                             action="store_true", help="do not stop build on error") 
608          parser.add_option("--build-everything", dest="robust", default=False, 
609                             action="store_true", help="build all packages regardless of errors") 
610          parser.add_option("-V", dest="full_verbose", default=False, 
611                            action="store_true", help="display all builds") 
612          parser.add_option("-s", "--specified-only", dest="specified_only", default=False, 
613                            action="store_true", help="only build packages specified on the command line") 
614          parser.add_option("--buildtest", dest="buildtest", 
615                            action="append", help="package to buildtest") 
616          parser.add_option("--buildtest1", dest="buildtest1", 
617                            action="append", help="package to buildtest1") 
618          parser.add_option("--output", dest="output_dir", 
619                            action="store", help="where to output results") 
620          parser.add_option("--pre-clean", dest="pre_clean", 
621                            action="store_true", help="run make clean first") 
622          parser.add_option("--bootstrap", dest="bootstrap", default=False, 
623                            action="store_true", help="DEPRECATED, UNUSED") 
624          parser.add_option("--disable-logging", dest="logging_enabled", default=True, 
625                            action="store_false", help="turn off all logs") 
626          parser.add_option("--target", dest="target", 
627                            action="store", help="run make with this target") 
628          parser.add_option("--pjobs", dest="ros_parallel_jobs", type="int", 
629                            action="store", help="Override ROS_PARALLEL_JOBS environment variable with this number of jobs.") 
630          parser.add_option("--threads", dest="threads", type="int", default = os.environ.get("ROSMAKE_THREADS", parallel_build.num_cpus()), 
631                            action="store", help="Build up to N packages in parallel") 
632          parser.add_option("--profile", dest="print_profile", default=False, 
633                            action="store_true", help="print time profile after build") 
634          parser.add_option("--skip-blacklist", dest="skip_blacklist",  
635                            default=False, action="store_true",  
636                            help="skip packages containing a file called ROS_BUILD_BLACKLIST (Default behavior will ignore the presence of ROS_BUILD_BLACKLIST)") 
637          parser.add_option("--skip-blacklist-osx", dest="skip_blacklist_osx",  
638                            default=False, action="store_true",  
639                            help="deprecated option. it will do nothing, please use platform declarations and --require-platform instead") 
640   
641          parser.add_option("--status-rate", dest="status_update_rate", 
642                            action="store", help="How fast to update the status bar in Hz.  Default: 5Hz") 
643           
644   
645          options, args = parser.parse_args() 
646          self.printer.print_all('rosmake starting...') 
647   
648          rospack = self.rospack 
649          rosstack = self.rosstack 
650   
651          testing = False 
652          building = True 
653          if options.test_only: 
654              testing = True 
655              building = False 
656          elif options.test: 
657              testing = True 
658   
659          if options.ros_parallel_jobs: 
660              self.ros_parallel_jobs = options.ros_parallel_jobs 
661   
662          self.robust_build = options.robust 
663          self.best_effort = options.best_effort 
664          self.threads = options.threads 
665          self.skip_blacklist = options.skip_blacklist 
666          if options.skip_blacklist_osx: 
667              self.printer.print_all("Option --skip-blacklist-osx is deprecated. It will do nothing, please use platform declarations and --require-platform instead"); 
668          self.logging_enabled = options.logging_enabled 
669   
670          # pass through verbosity options 
671          self.printer.full_verbose = options.full_verbose 
672          self.printer.verbose = options.verbose 
673          if options.status_update_rate: 
674              if float(options.status_update_rate)> 0: 
675                  self.printer.duration = 1.0/float(options.status_update_rate) 
676              else: 
677                  self.printer.duration = 0 
678   
679          packages = [] 
680          #load packages from arguments 
681          if options.build_all: 
682              packages = [x for x in rospack.list() if not self.rospack.get_manifest(x).is_catkin] 
683              self.printer.print_all( "Building all packages") 
684          else:      # no need to extend if all already selected    
685              if options.buildtest: 
686                for p in options.buildtest: 
687                  packages.extend(self.rospack.get_depends_on(p))  
688                  self.printer.print_all( "buildtest requested for package %s adding it and all dependent packages: "%p) 
689   
690              if options.buildtest1: 
691                for p in options.buildtest1: 
692                  packages.extend(self.rospack.get_depends_on(p, implicit=False))  
693                  self.printer.print_all( "buildtest1 requested for package %s adding it and all depends-on1 packages: "%p) 
694   
695          if len(packages) == 0 and len(args) == 0: 
696              p = os.path.basename(os.path.abspath('.')) 
697              try: 
698                if os.path.samefile(rospack.get_path(p), '.'): 
699                  packages = [p] 
700                  self.printer.print_all( "No package specified.  Building %s"%packages) 
701                else: 
702                  self.printer.print_all("No package selected and the current directory is not the correct path for package '%s'."%p) 
703                   
704              except rospkg.ResourceNotFound as ex: 
705                  try: 
706                      stack_dir = rosstack.get_path(p) 
707                      if os.path.samefile(stack_dir, '.'): 
708                          packages = [p] 
709                          self.printer.print_all( "No package specified.  Building stack %s"%packages) 
710                      else: 
711                          self.printer.print_all("No package or stack arguments and the current directory is not the correct path for stack '%s'. Stack directory is: %s."%(p, rosstack.get_path(p))) 
712                  except: 
713                      self.printer.print_all("No package or stack specified.  And current directory '%s' is not a package name or stack name."%p) 
714          else: 
715              packages.extend(args) 
716   
717          self.printer.print_all( "Packages requested are: %s"%packages) 
718           
719          # Setup logging 
720          if self.logging_enabled: 
721            date_time_stamp =  "rosmake_output-" + time.strftime("%Y%m%d-%H%M%S") 
722            if options.output_dir: 
723                #self.log_dir = os.path.join(os.getcwd(), options.output_dir, date_time_stamp); 
724                self.log_dir = os.path.abspath(options.output_dir) 
725            else: 
726                self.log_dir = os.path.join(rospkg.get_ros_home(), "rosmake", date_time_stamp); 
727   
728            self.printer.print_all("Logging to directory %s"%self.log_dir) 
729            if os.path.exists (self.log_dir) and not os.path.isdir(self.log_dir): 
730                self.printer.print_all( "Log destination %s is a file; please remove it or choose a new destination"%self.log_dir) 
731                sys.exit(1) 
732            if not os.path.exists (self.log_dir): 
733                self.printer.print_verbose("%s doesn't exist: creating"%self.log_dir) 
734                makedirs_with_parent_perms(self.log_dir) 
735   
736            self.printer.print_verbose("Finished setting up logging") 
737   
738          stacks_arguments = [s for s in packages if s in rosstack.list()] 
739          (self.specified_packages, self.rejected_packages) = rospkg.expand_to_packages(packages, rospack, rosstack) 
740   
741          self.printer.print_all("Expanded args %s to:\n%s"%(packages, self.specified_packages)) 
742          if self.rejected_packages: 
743              self.printer.print_all("WARNING: The following args could not be parsed as stacks or packages: %s"%self.rejected_packages) 
744          if len(self.specified_packages) + len(stacks_arguments) == 0: 
745              self.printer.print_all("ERROR: No arguments could be parsed into valid package or stack names.") 
746              self.printer.running = False 
747              return False 
748   
749          if options.unmark_installed: 
750              for p in self.specified_packages: 
751                  if self.flag_tracker.remove_nobuild(p): 
752                      self.printer.print_all("Removed ROS_NOBUILD from %s"%p) 
753              self.printer.running = False 
754              return True 
755               
756          required_packages = self.specified_packages[:] 
757   
758          # catch dependent packages which are inside of zero sized stacks #3528  
759          # add them to required list but not the specified list.  
760          for s in stacks_arguments: 
761              for d in rosstack.get_depends(s, implicit=False): 
762                  required_packages.extend(rosstack.packages_of(d)) 
763   
764          # deduplicate required_packages 
765          required_packages = list(set(required_packages)) 
766   
767          # make sure all dependencies are satisfied and if not warn 
768          buildable_packages = [] 
769          for p in required_packages: 
770              (buildable, error, str) = self.flag_tracker.can_build(p, self.skip_blacklist, [], False) 
771              if buildable:  
772                  buildable_packages.append(p) 
773   
774          #generate the list of packages necessary to build(in order of dependencies) 
775          counter = 0 
776          for p in required_packages: 
777   
778              counter = counter + 1 
779              self.printer.print_verbose( "Processing %s and all dependencies(%d of %d requested)"%(p, counter, len(packages))) 
780              self.build_or_recurse(p) 
781   
782          # remove extra packages if specified-only flag is set 
783          if options.specified_only: 
784            new_list = [] 
785            for pkg in self.build_list: 
786              if pkg in self.specified_packages: 
787                new_list.append(pkg) 
788                self.dependency_tracker = parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack) # this will make the tracker only respond to packages in the list 
789           
790            self.printer.print_all("specified-only option was used, only building packages %s"%new_list) 
791            self.build_list = new_list 
792   
793          if options.pre_clean: 
794            build_queue = parallel_build.BuildQueue(self.build_list, parallel_build.DependencyTracker([], rospack=self.rospack), robust_build = True) 
795            self.parallel_build_pkgs(build_queue, "clean", threads = options.threads) 
796   
797          build_passed = True 
798   
799          if building: 
800            self.printer.print_verbose ("Building packages %s"% self.build_list) 
801            build_queue = parallel_build.BuildQueue(self.build_list, self.dependency_tracker, robust_build = options.robust or options.best_effort) 
802            if None not in self.result.keys(): 
803                  self.result[None] = {} 
804   
805            build_passed = self.parallel_build_pkgs(build_queue, options.target, threads = options.threads) 
806   
807          tests_passed = True 
808          if build_passed and testing: 
809              self.printer.print_verbose ("Testing packages %s"% packages) 
810              build_queue = parallel_build.BuildQueue(self.specified_packages, parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack), robust_build = True) 
811              tests_passed = self.parallel_build_pkgs(build_queue, "test", threads = 1) 
812   
813   
814          if  options.mark_installed: 
815              if build_passed and tests_passed:  
816                  for p in self.specified_packages: 
817                      if self.flag_tracker.add_nobuild(p): 
818                          self.printer.print_all("Marking %s as installed with a ROS_NOBUILD file"%p) 
819              else: 
820                  self.printer.print_all("All builds and tests did not pass cannot mark packages as installed. ") 
821   
822   
823          self.finish_time = time.time() #note: before profiling 
824          self.generate_summary_output(self.log_dir) 
825           
826          if options.print_profile: 
827              self.printer.print_all (self.get_profile_string()) 
828   
829          self.printer.running = False 
830          return build_passed and tests_passed 
831