Source Code for Module rosmake.engine

  1  #! /usr/bin/env python 
  2   
  3  # Copyright (c) 2009, Willow Garage, Inc. 
  4  # All rights reserved. 
  5  #  
  6  # Redistribution and use in source and binary forms, with or without 
  7  # modification, are permitted provided that the following conditions are met: 
  8  #  
  9  #     * Redistributions of source code must retain the above copyright 
 10  #       notice, this list of conditions and the following disclaimer. 
 11  #     * Redistributions in binary form must reproduce the above copyright 
 12  #       notice, this list of conditions and the following disclaimer in the 
 13  #       documentation and/or other materials provided with the distribution. 
 14  #     * Neither the name of Willow Garage, Inc. nor the names of its 
 15  #       contributors may be used to endorse or promote products derived from 
 16  #       this software without specific prior written permission. 
 17  #  
 18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
 19  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
 20  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 21  # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
 22  # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 23  # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 24  # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 25  # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 26  # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 27  # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 28  # POSSIBILITY OF SUCH DAMAGE. 
 29   
 30  # Author Tully Foote/tfoote@willowgarage.com 
 31   
 32  from __future__ import print_function 
 33   
 34  import os 
 35  import re 
 36  import signal 
 37  import sys 
 38  import subprocess 
 39  import time 
 40  import threading 
 41  import traceback 
 42   
 43  import rospkg 
 44  from rospkg import ResourceNotFound 
 45   
 46  try:  
 47      from exceptions import SystemExit #Python 2.x 
 48  except ImportError:  
 49      pass #Python 3.x (in Python 3, 'exceptions' is always imported)  
 50   
 51  from operator import itemgetter 
 52   
 53  from . import parallel_build 
 54  from . import package_stats 
 55   
 56  from optparse import OptionParser 
 57  from .gcc_output_parse import Warnings 
 58   
 59  # #3883 
 60  _popen_lock = threading.Lock() 
 61   
 62 -def make_command(): 
 63      """ 
 64      @return: name of 'make' command 
 65      @rtype: str 
 66      """ 
 67      return os.environ.get("MAKE", "make") 
 68   
 69  # this is a copy of the roslogging utility. it's been moved here as it is a common 
 70  # routine for programs using accessing ROS directories 
 71 -def makedirs_with_parent_perms(p): 
 72      """ 
 73      Create the directory using the permissions of the nearest 
 74      (existing) parent directory. This is useful for logging, where a 
 75      root process sometimes has to log in the user's space. 
 76      @param p: directory to create 
 77      @type  p: str 
 78      """     
 79      p = os.path.abspath(p) 
 80      parent = os.path.dirname(p) 
 81      # recurse upwards, checking to make sure we haven't reached the 
 82      # top 
 83      if not os.path.exists(p) and p and parent != p: 
 84          makedirs_with_parent_perms(parent) 
 85          s = os.stat(parent) 
 86          os.mkdir(p) 
 87   
 88          # if perms of new dir don't match, set anew 
 89          s2 = os.stat(p) 
 90          if s.st_uid != s2.st_uid or s.st_gid != s2.st_gid: 
 91              os.chown(p, s.st_uid, s.st_gid) 
 92          if s.st_mode != s2.st_mode: 
 93              os.chmod(p, s.st_mode)     
 94   
 95 -class Printer: 
 96     # storage for the instance reference 
 97      __instance = None 
 98   
 99 -    def __init__(self): 
100          """ Create singleton instance """ 
101          # Check whether we already have an instance 
102          if Printer.__instance is None: 
103              # Create and remember instance 
104              Printer.__instance = Printer.__impl() 
105   
106          # Store instance reference as the only member in the handle 
107          self.__dict__['_Printer__instance'] = Printer.__instance 
108   
109 -    def __getattr__(self, attr): 
110          """ Delegate access to implementation """ 
111          return getattr(self.__instance, attr) 
112   
113 -    def __setattr__(self, attr, value): 
114          """ Delegate access to implementation """ 
115          return setattr(self.__instance, attr, value) 
116       
117 -    def __enter__(self): 
118          """Pass through for the __enter__ function for the __instance""" 
119          return self.__instance.__enter__() 
120       
121 -    def __exit__(self, mtype, value, tb): 
122          """Pass through for the __exit__ function for the __instance""" 
123          return self.__instance.__exit__(mtype, value, tb) 
124       
125 -    class __impl(threading.Thread): 
126 -        def __init__(self): 
127              threading.Thread.__init__(self) 
128              self.build_queue = None 
129              self.condition = threading.Condition() 
130              self.running = True 
131              self.done = False 
132              self.status = "" 
133              self.verbose = False 
134              self.full_verbose = False 
135              self.duration = 1./10. 
136              self._last_status = None 
137   
138              # Rosmake specific data 
139              self.cache_argument = None 
140              self.cache_right = '' 
141              self.pkg_start_times = {} 
142   
143 -        def shutdown(self): 
144              self.running = False 
145              cycles = 10 
146              for i in range(0,cycles):# sleep for at least 2 cycles of the status testing 'cycles' times 
147                  if self.done: 
148                      #print "SUCCESSFULLY SHUTDOWN" 
149                      return True 
150                  #print "Sleeping for %f FOR SHUTDOWN.  %d threads running"%(max(self.duration/cycles*2, 0.01), threading.activeCount()) 
151                  time.sleep(max(self.duration, 0.1)/cycles*2)  
152              raise Exception("Failed to shutdown status thread in %.2f seconds"%(self.duration * 2)) 
153   
154 -        def __enter__(self): 
155              self.start() 
156 -        def __exit__(self, mtype, value, tb): 
157              self.shutdown() 
158              if value: 
159                  if not mtype == type(SystemExit()): 
160                      traceback.print_exception(mtype, value, tb) 
161                  else: 
162                      sys.exit(value) 
163   
164 -        def run(self): 
165              while self.running: 
166                  #shutdown if duration set to zero 
167                  if self.duration <= 0: 
168                      self.running = False 
169                      break 
170                  self.set_status_from_cache() 
171                  if len(self.pkg_start_times.keys()) > 0: 
172                      n = self.terminal_width() - len(self.status) 
173                      status = self.status 
174                      if n > 0: 
175                          status = " "*n + self.status 
176                      if status != self._last_status: 
177                          self._print_status("%s"%status) 
178                          self._last_status = status 
179                  time.sleep(self.duration)  
180              self.done = True 
181              #print "STATUS THREAD FINISHED" 
182   
183 -        def rosmake_cache_info(self, argument, start_times, right): 
184              self.cache_argument = argument 
185              self.pkg_start_times = start_times 
186              self.cache_right = right 
187   
188 -        def rosmake_pkg_times_to_string(self, start_times): 
189              threads = [] 
190              for p, t in sorted(start_times.items(), key=itemgetter(1)): #py3k 
191                  threads.append("[ %s: %.1f sec ]"%(p, time.time() - t)) 
192   
193              return " ".join(threads) 
194   
195 -        def set_status_from_cache(self): 
196              if self.cache_argument: 
197                  self.set_status("[ make %s ] "%self.cache_argument + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
198              else: 
199                  self.set_status("[ make ] " + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
200   
201 -        def set_status(self, left, right = ''): 
202              header = "[ rosmake ] " 
203              h = len(header) 
204              l = len(left) 
205              r = len(right) 
206              w = self.terminal_width() 
207              if l + r < w - h: 
208                  padding = w - h - l - r 
209                  self.status = header + left + " "*padding + right 
210              else: 
211                  self.status = header + left[:(w - h - r - 4)] + "... " + right 
212   
213 -        def print_all(self, s, thread_name=None): 
214              if thread_name is None: 
215                  str = "[ rosmake ] %s"%s 
216                   
217   
218              else: 
219                  str = "[rosmake-%s] %s"%(thread_name, s) 
220              sys.stdout.write(self.pad_str_to_width(str, self.terminal_width())+"\n") 
221              sys.stdout.flush() 
222   
223 -        def print_verbose(self, s, thread_name=None): 
224              if self.verbose or self.full_verbose: 
225                  self.print_all(s, thread_name=thread_name) 
226   
227 -        def print_full_verbose(self, s): 
228              if self.full_verbose: 
229                  print("[ rosmake ] %s"%(s)) 
230   
231 -        def print_tail(self, s, tail_lines=40): 
232              lines = s.splitlines() 
233              if self.full_verbose: 
234                  tail_lines = len(lines) 
235   
236              num_lines = min(len(lines), tail_lines) 
237              if num_lines == tail_lines: 
238                  print("[ rosmake ] Last %d lines"%(num_lines)) 
239              else: 
240                  print("[ rosmake ] All %d lines"%(num_lines)) 
241              print("{" + "-"*79) 
242              for l in range(-num_lines, -1): 
243                  print("  %s"%(lines[l])) 
244              print("-"*79 + "}") 
245   
246 -        def _print_status(self, s): 
247              sys.stdout.write("%s\r"%(s)) 
248              sys.stdout.flush() 
249   
250          @staticmethod 
251 -        def terminal_width(): 
252              """Estimate the width of the terminal""" 
253              width = 0 
254              try: 
255                  import struct, fcntl, termios 
256                  s = struct.pack('HHHH', 0, 0, 0, 0) 
257                  x = fcntl.ioctl(1, termios.TIOCGWINSZ, s) 
258                  width = struct.unpack('HHHH', x)[1] 
259              except ImportError: 
260                  pass 
261              except IOError: 
262                  pass 
263              if width <= 0: 
264                  try: 
265                      width = int(os.environ['COLUMNS']) 
266                  except: 
267                      pass 
268              if width <= 0: 
269                  width = 80 
270   
271              return width 
272   
273          @staticmethod 
274 -        def pad_str_to_width(str, width): 
275              """ Pad the string to be terminal width""" 
276              length = len(str) 
277              excess = 0 
278              if length < width: 
279                  excess = width - length 
280              return str + " "* excess 
281   
282   
283   
284 -class RosMakeAll: 
285 -    def __init__(self): 
286          self._result_lock = threading.Lock() 
287   
288          self.rospack = rospkg.RosPack() 
289          self.rosstack = rospkg.RosStack() 
290   
291          self.printer = Printer() 
292          self.result = {} 
293          self.paths = {} 
294          self.dependency_tracker = parallel_build.DependencyTracker(rospack=self.rospack) 
295          self.flag_tracker = package_stats.PackageFlagTracker(self.dependency_tracker) 
296          self.output = {} 
297          self.profile = {} 
298          self.ros_parallel_jobs = 0 
299          self.build_list = [] 
300          self.start_time = time.time() 
301          self.log_dir = "" 
302          self.logging_enabled = True 
303   
304 -    def num_packages_built(self): 
305          """ 
306          @return: number of packages that were built 
307          @rtype: int 
308          """ 
309          return len(list(self.result[argument].keys())) #py3k 
310   
311 -    def update_status(self, argument, start_times, right): 
312          self.printer.rosmake_cache_info(argument, start_times, right) 
313   
314 -    def build_or_recurse(self,p): 
315          if p in self.build_list: 
316              return 
317          for d in self.dependency_tracker.get_deps_1(p): 
318              self.build_or_recurse(d) 
319          try: # append it ot the list only if present 
320            self.rospack.get_path(p) 
321            self.build_list.append(p) 
322          except rospkg.ResourceNotFound as ex: 
323            if not self.robust_build: 
324              self.printer.print_all("Exiting due to missing package: %s"%ex) 
325              sys.exit(-1) 
326            else: 
327              self.printer.print_all("!"*20 + " Package %s does not exist. %s"%(p, ex) + "!"*20) 
328   
329   
330 -    def parallel_build_pkgs(self, build_queue, argument = None, threads = 1): 
331          self.profile[argument] = {} 
332          self.output[argument] = {} 
333          with self._result_lock: 
334              if argument not in self.result.keys(): 
335                  self.result[argument] = {} 
336   
337          cts = [] 
338          for i in range(0, threads): 
339            ct = parallel_build.CompileThread(str(i), build_queue, self, argument) 
340            #print "TTTH starting thread ", ct 
341            ct.start() 
342            cts.append(ct) 
343          for ct in cts: 
344            try: 
345              #print "TTTT Joining", ct 
346              ct.join() 
347              #print "TTTH naturally ended thread", ct 
348            except KeyboardInterrupt: 
349              self.printer.print_all( "TTTH Caught KeyboardInterrupt. Stopping build.") 
350              build_queue.stop() 
351              ct.join() 
352            except: #catch all 
353                self.printer.print_all("TTTH OTHER exception thrown!!!!!!!!!!!!!!!!!!!!!") 
354                ct.join() 
355          #print "All threads joined" 
356          all_pkgs_passed = True 
357          with self._result_lock: 
358              for v in self.result[argument].values(): 
359                  all_pkgs_passed = v and all_pkgs_passed 
360   
361          build_passed = build_queue.succeeded() and all_pkgs_passed 
362          return build_passed 
363   
364      # This function taken from 
365      # http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html 
366 -    def _subprocess_setup(self): 
367          # Python installs a SIGPIPE handler by default. This is usually not 
368          # what non-Python subprocesses expect. 
369          signal.signal(signal.SIGPIPE, signal.SIG_DFL) 
370   
371 -    def _build_package(self, package, argument=None): 
372          """ 
373          Lower-level routine for building a package. Handles execution of actual build command. 
374          @param package: package name 
375          @type  package: str 
376          """ 
377          local_env = os.environ.copy() 
378          if self.ros_parallel_jobs > 0: 
379              local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (self.ros_parallel_jobs, self.ros_parallel_jobs) 
380          elif "ROS_PARALLEL_JOBS" not in os.environ: #if no environment setup and no args fall back to # cpus 
381              # num_cpus check can (on OS X) trigger a Popen(), which has 
382              #the multithreading bug we wish to avoid on Py2.7. 
383              with _popen_lock: 
384                  num_cpus = parallel_build.num_cpus() 
385                  local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (num_cpus, num_cpus) 
386          local_env['SVN_CMDLINE'] = "svn --non-interactive" 
387          cmd = ["bash", "-c", "cd %s && %s "%(self.rospack.get_path(package), make_command()) ] #UNIXONLY 
388          if argument: 
389              cmd[-1] += argument 
390          self.printer.print_full_verbose (cmd) 
391          # #3883: make sure only one Popen command occurs at a time due to 
392          # http://bugs.python.org/issue13817 
393          with _popen_lock: 
394              command_line = subprocess.Popen(cmd, stdout=subprocess.PIPE,  stderr=subprocess.STDOUT, env=local_env, preexec_fn=self._subprocess_setup) 
395          (pstd_out, pstd_err) = command_line.communicate() # pstd_err should be None due to pipe above 
396          if not isinstance(pstd_out, str): 
397              pstd_out = pstd_out.decode() 
398          return (command_line.returncode, pstd_out) 
399   
400 -    def build(self, p, argument = None, robust_build=False): 
401          """ 
402          Build package 
403          @param p: package name 
404          @type  p: str 
405          """ 
406          return_string = "" 
407          try: 
408              # warn if ROS_BUILD_BLACKLIST encountered if applicable 
409              # do not build packages for which the build has failed 
410              if argument == "test":  # Tests are not build dependent 
411                  failed_packages = [] 
412              else: 
413                  with self._result_lock: 
414                      failed_packages = [j for j in self.result[argument] if not self.result[argument][j] == True] 
415   
416              (buildable, error, why) = self.flag_tracker.can_build(p, self.skip_blacklist, failed_packages) 
417              if buildable or self.robust_build: 
418                  start_time = time.time() 
419                  (returncode, pstd_out) = self._build_package(p, argument) 
420                  self.profile[argument][p] = time.time() - start_time 
421                  self.output[argument][p] = pstd_out 
422                  if argument: 
423                      log_type = "build_%s"%argument 
424                  else: 
425                      log_type = "build" 
426                  if not returncode: 
427                      self.printer.print_full_verbose( pstd_out) 
428                      with self._result_lock: 
429                          self.result[argument][p] = True 
430                      warnings = Warnings( pstd_out ) 
431                      num_warnings = len( warnings.warning_lines ) 
432                      if num_warnings > 0: 
433                          return_string =  "[PASS] [ %.2f seconds ] [ %d warnings "%(self.profile[argument][p], num_warnings) 
434                          warning_dict = warnings.analyze(); 
435                          for warntype,warnlines in warning_dict.items(): 
436                              if len( warnlines ) > 0: 
437                                  return_string = return_string + '[ {0:d} {1} ] '.format(len(warnlines),warntype) 
438                          return_string = return_string + ' ]' 
439                      else: 
440                          return_string =  ("[PASS] [ %.2f seconds ]"%( self.profile[argument][p])) 
441                      self.output_to_file(p, log_type, pstd_out, num_warnings > 0) 
442                  else: 
443                      success = False 
444                      no_target = len(re.findall("No rule to make target", pstd_out)) > 0 
445                      interrupt = len(re.findall("Interrupt", pstd_out)) > 0 
446                      if no_target: 
447                          return_string = ( "[SKIP] No rule to make target %s"%( argument)) 
448                          success = True 
449                      elif interrupt: 
450                          return_string = ("[Interrupted]" ) 
451                      else: 
452                          return_string = ( "[FAIL] [ %.2f seconds ]"%( self.profile[argument][p])) 
453                      with self._result_lock: 
454                          self.result[argument][p] = True if no_target else False 
455   
456                      if success == False: #don't print tail if [SKIP] target 
457                          self.printer.print_tail( pstd_out) 
458                      self.output_to_file(p, log_type, pstd_out, always_print= not (no_target or interrupt)) 
459   
460                      return (success, return_string) 
461              else: 
462                  with self._result_lock: 
463                      self.result[argument][p] = error 
464   
465                  return_string += why 
466                  return(error, return_string) 
467              return (True, return_string) # this means that we didn't error in any case above 
468          except rospkg.ResourceNotFound as ex: 
469              with self._result_lock: 
470                  self.result[argument][p] = False 
471              self.printer.print_verbose ("[SKIP] Package %s not found\n" % p) 
472              self.output[argument][p] = "Package not found %s"%ex 
473              return (False, return_string) 
474               
475   
476 -    def output_to_file(self, package, log_type, stdout, always_print= False): 
477          if not self.logging_enabled: 
478              return 
479          package_log_dir = os.path.join(self.log_dir, package) 
480   
481          std_out_filename = os.path.join(package_log_dir, log_type + "_output.log") 
482          if not os.path.exists (package_log_dir): 
483              makedirs_with_parent_perms(package_log_dir) 
484          with open(std_out_filename, 'w') as stdout_file: 
485              stdout_file.write(stdout) 
486              print_string = "Output from build of package %s written to:\n[ rosmake ]    %s"%(package, std_out_filename) 
487              if always_print: 
488                  self.printer.print_all(print_string) 
489              else: 
490                  self.printer.print_full_verbose(print_string) 
491   
492 -    def generate_summary_output(self, log_dir): 
493          if not self.logging_enabled: 
494              return 
495   
496          self.printer.print_all("Results:") 
497          if 'clean' in self.result.keys(): 
498              self.printer.print_all("Cleaned %d packages."%len(self.result['clean'])) 
499          if None in self.result.keys(): 
500              build_failure_count = len([p for p in self.result[None].keys() if self.result[None][p] == False]) 
501              self.printer.print_all("Built %d packages with %d failures."%(len(self.result[None]), build_failure_count)) 
502          if 'test' in self.result.keys(): 
503              test_failure_count = len([p for p in self.result['test'].keys() if self.result['test'][p] == False]) 
504              self.printer.print_all("Tested %d packages with %d failures."%(len(self.result['test']), test_failure_count)) 
505          self.printer.print_all("Summary output to directory") 
506          self.printer.print_all("%s"%self.log_dir) 
507          if self.rejected_packages: 
508              self.printer.print_all("WARNING: Skipped command line arguments: %s because they could not be resolved to a stack name or a package name. "%self.rejected_packages) 
509   
510                              
511   
512          if None in self.result.keys(): 
513              if len(self.result[None].keys()) > 0: 
514                  buildfail_filename = os.path.join(log_dir, "buildfailures.txt") 
515                  with open(buildfail_filename, 'w') as bf: 
516                      bf.write("Build failures:\n") 
517                      for key in self.build_list: 
518                          if key in self.result[None].keys() and self.result[None][key] == False: 
519                              bf.write("%s\n"%key) 
520                  if  None in self.output.keys(): 
521                      buildfail_context_filename = os.path.join(log_dir, "buildfailures-with-context.txt") 
522                      with open(buildfail_context_filename, 'w') as bfwc: 
523                          bfwc.write("Build failures with context:\n") 
524                          for key in self.build_list: 
525                              if key in self.result[None].keys() and self.result[None][key] == False: 
526                                  bfwc.write("---------------------\n") 
527                                  bfwc.write("%s\n"%key) 
528                                  if key in self.output[None]: 
529                                      bfwc.write(self.output[None][key]) 
530   
531          if "test" in self.result.keys(): 
532              if len(self.result["test"].keys()) > 0: 
533                  testfail_filename = os.path.join(log_dir, "testfailures.txt") 
534                  with open(testfail_filename, 'w') as btwc: 
535                      btwc.write("Test failures:\n") 
536                      for key in self.build_list: 
537                          if key in self.result["test"].keys() and self.result["test"][key] == False: 
538                              btwc.write("%s\n"%key) 
539   
540                  if "test" in self.output.keys(): 
541                      testfail_filename = os.path.join(log_dir, "testfailures-with-context.txt") 
542                      with open(testfail_filename, 'w') as btwc: 
543                          btwc.write("Test failures with context:\n") 
544                          for key in self.build_list: 
545                              if key in self.result["test"].keys() and self.result["test"][key] == False: 
546                                  btwc.write("%s\n"%key) 
547                                  if key in self.output["test"]: 
548                                      btwc.write(self.output["test"][key]) 
549   
550          profile_filename = os.path.join(log_dir, "profile.txt") 
551          with open(profile_filename, 'w') as pf: 
552              pf.write(self.get_profile_string()) 
553                               
554                               
555   
556 -    def get_profile_string(self): 
557          output = '--------------\nProfile\n--------------\n' 
558          total = 0.0 
559          count = 1 
560          for key in self.build_list: 
561              build_results = ["[Not Built ]", "[  Built   ]", "[Build Fail]"]; 
562              test_results =  ["[Untested ]", "[Test Pass]", "[Test Fail]"]; 
563              build_result = 0 
564              test_result = 0 
565              test_time = 0.0 
566              build_time = 0.0 
567   
568              if None in self.result.keys(): 
569                  if key in self.result[None].keys(): 
570                      if self.result[None][key] == True: 
571                          build_result = 1 
572                      else: 
573                          build_result = 2 
574   
575              if "test" in self.profile.keys(): 
576                  if key in self.result["test"].keys(): 
577                      if self.result["test"][key] == True: 
578                          test_result = 1 
579                      else: 
580                          test_result = 2 
581   
582              if None in self.profile.keys(): 
583                  if key in self.profile[None].keys(): 
584                      build_time = self.profile[None][key] 
585   
586              if "test" in self.profile.keys(): 
587                  if key in self.profile["test"].keys(): 
588                      test_time = self.profile["test"][key] 
589                   
590                       
591              output = output + "%3d: %s in %.2f %s in %.2f --- %s\n"% (count, build_results[build_result], build_time , test_results[test_result], test_time, key) 
592              total = total + build_time 
593              count = count + 1 
594   
595          elapsed_time  =  self.finish_time - self.start_time 
596          output = output + "----------------\n" + "%.2f Cumulative,  %.2f Elapsed, %.2f Speedup \n"%(total, elapsed_time, float(total) / float(elapsed_time)) 
597          return output 
598   
599 -    def main(self): 
600          """ 
601          main command-line entrypoint 
602          """ 
603          parser = OptionParser(usage="usage: %prog [options] [PACKAGE]...", 
604                                description="rosmake recursively builds all dependencies before building a package", prog='rosmake') 
605          parser.add_option("--test-only", dest="test_only", default=False, 
606                            action="store_true", help="only run tests") 
607          parser.add_option("-t", dest="test", default=False, 
608                            action="store_true", help="build and test packages") 
609          parser.add_option("-a", "--all", dest="build_all", default=False, 
610                            action="store_true", help="select all packages") 
611          parser.add_option("-i", "--mark-installed", dest="mark_installed", default=False, 
612                            action="store_true", help="On successful build, mark specified packages as installed with ROS_NOBUILD") 
613          parser.add_option("-u", "--unmark-installed", dest="unmark_installed", default=False, 
614                            action="store_true", help="Remove ROS_NOBUILD from the specified packages.  This will not build anything.") 
615          parser.add_option("-v", dest="verbose", default=False, 
616                            action="store_true", help="display errored builds") 
617          parser.add_option("-r","-k", "--robust", dest="best_effort", default=False, 
618                             action="store_true", help="do not stop build on error") 
619          parser.add_option("--build-everything", dest="robust", default=False, 
620                             action="store_true", help="build all packages regardless of errors") 
621          parser.add_option("-V", dest="full_verbose", default=False, 
622                            action="store_true", help="display all builds") 
623          parser.add_option("-s", "--specified-only", dest="specified_only", default=False, 
624                            action="store_true", help="only build packages specified on the command line") 
625          parser.add_option("--buildtest", dest="buildtest", 
626                            action="append", help="package to buildtest") 
627          parser.add_option("--buildtest1", dest="buildtest1", 
628                            action="append", help="package to buildtest1") 
629          parser.add_option("--output", dest="output_dir", 
630                            action="store", help="where to output results") 
631          parser.add_option("--pre-clean", dest="pre_clean", 
632                            action="store_true", help="run make clean first") 
633          parser.add_option("--bootstrap", dest="bootstrap", default=False, 
634                            action="store_true", help="DEPRECATED, UNUSED") 
635          parser.add_option("--disable-logging", dest="logging_enabled", default=True, 
636                            action="store_false", help="turn off all logs") 
637          parser.add_option("--target", dest="target", 
638                            action="store", help="run make with this target") 
639          parser.add_option("--pjobs", dest="ros_parallel_jobs", type="int", 
640                            action="store", help="Override ROS_PARALLEL_JOBS environment variable with this number of jobs.") 
641          parser.add_option("--threads", dest="threads", type="int", default = os.environ.get("ROSMAKE_THREADS", parallel_build.num_cpus()), 
642                            action="store", help="Build up to N packages in parallel") 
643          parser.add_option("--profile", dest="print_profile", default=False, 
644                            action="store_true", help="print time profile after build") 
645          parser.add_option("--skip-blacklist", dest="skip_blacklist",  
646                            default=False, action="store_true",  
647                            help="skip packages containing a file called ROS_BUILD_BLACKLIST (Default behavior will ignore the presence of ROS_BUILD_BLACKLIST)") 
648          parser.add_option("--skip-blacklist-osx", dest="skip_blacklist_osx",  
649                            default=False, action="store_true",  
650                            help="deprecated option. it will do nothing, please use platform declarations and --require-platform instead") 
651   
652          parser.add_option("--status-rate", dest="status_update_rate", 
653                            action="store", help="How fast to update the status bar in Hz.  Default: 5Hz") 
654           
655   
656          options, args = parser.parse_args() 
657          self.printer.print_all('rosmake starting...') 
658   
659          rospack = self.rospack 
660          rosstack = self.rosstack 
661   
662          testing = False 
663          building = True 
664          if options.test_only: 
665              testing = True 
666              building = False 
667          elif options.test: 
668              testing = True 
669   
670          if options.ros_parallel_jobs: 
671              self.ros_parallel_jobs = options.ros_parallel_jobs 
672   
673          self.robust_build = options.robust 
674          self.best_effort = options.best_effort 
675          self.threads = options.threads 
676          self.skip_blacklist = options.skip_blacklist 
677          if options.skip_blacklist_osx: 
678              self.printer.print_all("Option --skip-blacklist-osx is deprecated. It will do nothing, please use platform declarations and --require-platform instead"); 
679          self.logging_enabled = options.logging_enabled 
680   
681          # pass through verbosity options 
682          self.printer.full_verbose = options.full_verbose 
683          self.printer.verbose = options.verbose 
684          if options.status_update_rate: 
685              if float(options.status_update_rate)> 0: 
686                  self.printer.duration = 1.0/float(options.status_update_rate) 
687              else: 
688                  self.printer.duration = 0 
689   
690          packages = [] 
691          #load packages from arguments 
692          if options.build_all: 
693              packages = [x for x in rospack.list() if not self.rospack.get_manifest(x).is_catkin] 
694              self.printer.print_all( "Building all packages") 
695          else:      # no need to extend if all already selected    
696              if options.buildtest: 
697                for p in options.buildtest: 
698                  packages.extend(self.rospack.get_depends_on(p))  
699                  self.printer.print_all( "buildtest requested for package %s adding it and all dependent packages: "%p) 
700   
701              if options.buildtest1: 
702                for p in options.buildtest1: 
703                  packages.extend(self.rospack.get_depends_on(p, implicit=False))  
704                  self.printer.print_all( "buildtest1 requested for package %s adding it and all depends-on1 packages: "%p) 
705   
706          if len(packages) == 0 and len(args) == 0: 
707              p = os.path.basename(os.path.abspath('.')) 
708              try: 
709                if os.path.samefile(rospack.get_path(p), '.'): 
710                  packages = [p] 
711                  self.printer.print_all( "No package specified.  Building %s"%packages) 
712                else: 
713                  self.printer.print_all("No package selected and the current directory is not the correct path for package '%s'."%p) 
714                   
715              except rospkg.ResourceNotFound as ex: 
716                  try: 
717                      stack_dir = rosstack.get_path(p) 
718                      if os.path.samefile(stack_dir, '.'): 
719                          packages = [p] 
720                          self.printer.print_all( "No package specified.  Building stack %s"%packages) 
721                      else: 
722                          self.printer.print_all("No package or stack arguments and the current directory is not the correct path for stack '%s'. Stack directory is: %s."%(p, rosstack.get_path(p))) 
723                  except: 
724                      self.printer.print_all("No package or stack specified.  And current directory '%s' is not a package name or stack name."%p) 
725          else: 
726              packages.extend(args) 
727   
728          self.printer.print_all( "Packages requested are: %s"%packages) 
729           
730          # Setup logging 
731          if self.logging_enabled: 
732            date_time_stamp =  "rosmake_output-" + time.strftime("%Y%m%d-%H%M%S") 
733            if options.output_dir: 
734                #self.log_dir = os.path.join(os.getcwd(), options.output_dir, date_time_stamp); 
735                self.log_dir = os.path.abspath(options.output_dir) 
736            else: 
737                self.log_dir = os.path.join(rospkg.get_ros_home(), "rosmake", date_time_stamp); 
738   
739            self.printer.print_all("Logging to directory %s"%self.log_dir) 
740            if os.path.exists (self.log_dir) and not os.path.isdir(self.log_dir): 
741                self.printer.print_all( "Log destination %s is a file; please remove it or choose a new destination"%self.log_dir) 
742                sys.exit(1) 
743            if not os.path.exists (self.log_dir): 
744                self.printer.print_verbose("%s doesn't exist: creating"%self.log_dir) 
745                makedirs_with_parent_perms(self.log_dir) 
746   
747            self.printer.print_verbose("Finished setting up logging") 
748   
749          stacks_arguments = [s for s in packages if s in rosstack.list()] 
750          (self.specified_packages, self.rejected_packages) = rospkg.expand_to_packages(packages, rospack, rosstack) 
751   
752          self.printer.print_all("Expanded args %s to:\n%s"%(packages, self.specified_packages)) 
753          if self.rejected_packages: 
754              self.printer.print_all("WARNING: The following args could not be parsed as stacks or packages: %s"%self.rejected_packages) 
755          if len(self.specified_packages) + len(stacks_arguments) == 0: 
756              self.printer.print_all("ERROR: No arguments could be parsed into valid package or stack names.") 
757              self.printer.running = False 
758              return False 
759   
760          if options.unmark_installed: 
761              for p in self.specified_packages: 
762                  if self.flag_tracker.remove_nobuild(p): 
763                      self.printer.print_all("Removed ROS_NOBUILD from %s"%p) 
764              self.printer.running = False 
765              return True 
766               
767          required_packages = self.specified_packages[:] 
768   
769          # catch packages of dependent stacks when specified stack is zero-sized #3528 
770          # add them to required list but not the specified list.  
771          for s in stacks_arguments: 
772              if not rosstack.packages_of(s): 
773                  for d in rosstack.get_depends(s, implicit=False): 
774                      try: 
775                          required_packages.extend(rosstack.packages_of(d)) 
776                      except ResourceNotFound: 
777                          self.printer.print_all('WARNING: The stack "%s" was not found. We will assume it is using the new buildsystem and try to continue...' % d) 
778   
779          # deduplicate required_packages 
780          required_packages = list(set(required_packages)) 
781   
782          # make sure all dependencies are satisfied and if not warn 
783          buildable_packages = [] 
784          for p in required_packages: 
785              (buildable, error, str) = self.flag_tracker.can_build(p, self.skip_blacklist, [], False) 
786              if buildable:  
787                  buildable_packages.append(p) 
788   
789          #generate the list of packages necessary to build(in order of dependencies) 
790          counter = 0 
791          for p in required_packages: 
792   
793              counter = counter + 1 
794              self.printer.print_verbose( "Processing %s and all dependencies(%d of %d requested)"%(p, counter, len(packages))) 
795              self.build_or_recurse(p) 
796   
797          # remove extra packages if specified-only flag is set 
798          if options.specified_only: 
799            new_list = [] 
800            for pkg in self.build_list: 
801              if pkg in self.specified_packages: 
802                new_list.append(pkg) 
803                self.dependency_tracker = parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack) # this will make the tracker only respond to packages in the list 
804           
805            self.printer.print_all("specified-only option was used, only building packages %s"%new_list) 
806            self.build_list = new_list 
807   
808          if options.pre_clean: 
809            build_queue = parallel_build.BuildQueue(self.build_list, parallel_build.DependencyTracker([], rospack=self.rospack), robust_build = True) 
810            self.parallel_build_pkgs(build_queue, "clean", threads = options.threads) 
811   
812          build_passed = True 
813   
814          if building: 
815            self.printer.print_verbose ("Building packages %s"% self.build_list) 
816            build_queue = parallel_build.BuildQueue(self.build_list, self.dependency_tracker, robust_build = options.robust or options.best_effort) 
817            if None not in self.result.keys(): 
818                  self.result[None] = {} 
819   
820            build_passed = self.parallel_build_pkgs(build_queue, options.target, threads = options.threads) 
821   
822          tests_passed = True 
823          if build_passed and testing: 
824              self.printer.print_verbose ("Testing packages %s"% packages) 
825              build_queue = parallel_build.BuildQueue(self.specified_packages, parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack), robust_build = True) 
826              tests_passed = self.parallel_build_pkgs(build_queue, "test", threads = 1) 
827   
828   
829          if  options.mark_installed: 
830              if build_passed and tests_passed:  
831                  for p in self.specified_packages: 
832                      if self.flag_tracker.add_nobuild(p): 
833                          self.printer.print_all("Marking %s as installed with a ROS_NOBUILD file"%p) 
834              else: 
835                  self.printer.print_all("All builds and tests did not pass cannot mark packages as installed. ") 
836   
837   
838          self.finish_time = time.time() #note: before profiling 
839          self.generate_summary_output(self.log_dir) 
840           
841          if options.print_profile: 
842              self.printer.print_all (self.get_profile_string()) 
843   
844          self.printer.running = False 
845          return build_passed and tests_passed 
846