Source Code for Module rosmake.engine

  1  #! /usr/bin/env python 
  2   
  3  # Copyright (c) 2009, Willow Garage, Inc. 
  4  # All rights reserved. 
  5  #  
  6  # Redistribution and use in source and binary forms, with or without 
  7  # modification, are permitted provided that the following conditions are met: 
  8  #  
  9  #     * Redistributions of source code must retain the above copyright 
 10  #       notice, this list of conditions and the following disclaimer. 
 11  #     * Redistributions in binary form must reproduce the above copyright 
 12  #       notice, this list of conditions and the following disclaimer in the 
 13  #       documentation and/or other materials provided with the distribution. 
 14  #     * Neither the name of Willow Garage, Inc. nor the names of its 
 15  #       contributors may be used to endorse or promote products derived from 
 16  #       this software without specific prior written permission. 
 17  #  
 18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
 19  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
 20  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 21  # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
 22  # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 23  # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 24  # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 25  # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 26  # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 27  # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 28  # POSSIBILITY OF SUCH DAMAGE. 
 29   
 30  # Author Tully Foote/tfoote@willowgarage.com 
 31   
 32  from __future__ import print_function 
 33   
 34  import os 
 35  import re 
 36  import signal 
 37  import sys 
 38  import subprocess 
 39  import time 
 40  import threading 
 41  import traceback 
 42   
 43  import rospkg 
 44  from rospkg import ResourceNotFound 
 45   
 46  try:  
 47      from exceptions import SystemExit #Python 2.x 
 48  except ImportError:  
 49      pass #Python 3.x (in Python 3, 'exceptions' is always imported)  
 50   
 51  from operator import itemgetter 
 52   
 53  from . import parallel_build 
 54  from . import package_stats 
 55   
 56  from optparse import OptionParser 
 57  from gcc_output_parse import Warnings 
 58   
 59  # #3883 
 60  _popen_lock = threading.Lock() 
 61   
 62 -def make_command(): 
 63      """ 
 64      @return: name of 'make' command 
 65      @rtype: str 
 66      """ 
 67      return os.environ.get("MAKE", "make") 
 68   
 69  # this is a copy of the roslogging utility. it's been moved here as it is a common 
 70  # routine for programs using accessing ROS directories 
 71 -def makedirs_with_parent_perms(p): 
 72      """ 
 73      Create the directory using the permissions of the nearest 
 74      (existing) parent directory. This is useful for logging, where a 
 75      root process sometimes has to log in the user's space. 
 76      @param p: directory to create 
 77      @type  p: str 
 78      """     
 79      p = os.path.abspath(p) 
 80      parent = os.path.dirname(p) 
 81      # recurse upwards, checking to make sure we haven't reached the 
 82      # top 
 83      if not os.path.exists(p) and p and parent != p: 
 84          makedirs_with_parent_perms(parent) 
 85          s = os.stat(parent) 
 86          os.mkdir(p) 
 87   
 88          # if perms of new dir don't match, set anew 
 89          s2 = os.stat(p) 
 90          if s.st_uid != s2.st_uid or s.st_gid != s2.st_gid: 
 91              os.chown(p, s.st_uid, s.st_gid) 
 92          if s.st_mode != s2.st_mode: 
 93              os.chmod(p, s.st_mode)     
 94   
 95 -class Printer: 
 96     # storage for the instance reference 
 97      __instance = None 
 98   
 99 -    def __init__(self): 
100          """ Create singleton instance """ 
101          # Check whether we already have an instance 
102          if Printer.__instance is None: 
103              # Create and remember instance 
104              Printer.__instance = Printer.__impl() 
105   
106          # Store instance reference as the only member in the handle 
107          self.__dict__['_Printer__instance'] = Printer.__instance 
108   
109 -    def __getattr__(self, attr): 
110          """ Delegate access to implementation """ 
111          return getattr(self.__instance, attr) 
112   
113 -    def __setattr__(self, attr, value): 
114          """ Delegate access to implementation """ 
115          return setattr(self.__instance, attr, value) 
116       
117 -    def __enter__(self): 
118          """Pass through for the __enter__ function for the __instance""" 
119          return self.__instance.__enter__() 
120       
121 -    def __exit__(self, mtype, value, tb): 
122          """Pass through for the __exit__ function for the __instance""" 
123          return self.__instance.__exit__(mtype, value, tb) 
124       
125 -    class __impl(threading.Thread): 
126 -        def __init__(self): 
127              threading.Thread.__init__(self) 
128              self.build_queue = None 
129              self.condition = threading.Condition() 
130              self.running = True 
131              self.done = False 
132              self.status = "" 
133              self.verbose = False 
134              self.full_verbose = False 
135              self.duration = 1./10. 
136              self._last_status = None 
137   
138              # Rosmake specific data 
139              self.cache_argument = None 
140              self.cache_right = '' 
141              self.pkg_start_times = {} 
142   
143 -        def shutdown(self): 
144              self.running = False 
145              cycles = 10 
146              for i in range(0,cycles):# sleep for at least 2 cycles of the status testing 'cycles' times 
147                  if self.done: 
148                      #print "SUCCESSFULLY SHUTDOWN" 
149                      return True 
150                  #print "Sleeping for %f FOR SHUTDOWN.  %d threads running"%(max(self.duration/cycles*2, 0.01), threading.activeCount()) 
151                  time.sleep(max(self.duration, 0.1)/cycles*2)  
152              raise Exception("Failed to shutdown status thread in %.2f seconds"%(self.duration * 2)) 
153   
154 -        def __enter__(self): 
155              self.start() 
156 -        def __exit__(self, mtype, value, tb): 
157              self.shutdown() 
158              if value: 
159                  if not mtype == type(SystemExit()): 
160                      traceback.print_exception(mtype, value, tb) 
161                  else: 
162                      sys.exit(value) 
163   
164 -        def run(self): 
165              while self.running: 
166                  #shutdown if duration set to zero 
167                  if self.duration <= 0: 
168                      self.running = False 
169                      break 
170                  self.set_status_from_cache() 
171                  if len(self.pkg_start_times.keys()) > 0: 
172                      n = self.terminal_width() - len(self.status) 
173                      status = self.status 
174                      if n > 0: 
175                          status = " "*n + self.status 
176                      if status != self._last_status: 
177                          self._print_status("%s"%status) 
178                          self._last_status = status 
179                  time.sleep(self.duration)  
180              self.done = True 
181              #print "STATUS THREAD FINISHED" 
182   
183 -        def rosmake_cache_info(self, argument, start_times, right): 
184              self.cache_argument = argument 
185              self.pkg_start_times = start_times 
186              self.cache_right = right 
187   
188 -        def rosmake_pkg_times_to_string(self, start_times): 
189              threads = [] 
190              for p, t in sorted(start_times.items(), key=itemgetter(1)): #py3k 
191                  threads.append("[ %s: %.1f sec ]"%(p, time.time() - t)) 
192   
193              return " ".join(threads) 
194   
195 -        def set_status_from_cache(self): 
196              if self.cache_argument: 
197                  self.set_status("[ make %s ] "%self.cache_argument + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
198              else: 
199                  self.set_status("[ make ] " + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
200   
201 -        def set_status(self, left, right = ''): 
202              header = "[ rosmake ] " 
203              h = len(header) 
204              l = len(left) 
205              r = len(right) 
206              w = self.terminal_width() 
207              if l + r < w - h: 
208                  padding = w - h - l - r 
209                  self.status = header + left + " "*padding + right 
210              else: 
211                  self.status = header + left[:(w - h - r - 4)] + "... " + right 
212   
213 -        def print_all(self, s, thread_name=None): 
214              if thread_name is None: 
215                  str = "[ rosmake ] %s"%s 
216                   
217   
218              else: 
219                  str = "[rosmake-%s] %s"%(thread_name, s) 
220              sys.stdout.write(self.pad_str_to_width(str, self.terminal_width())+"\n") 
221              sys.stdout.flush() 
222   
223 -        def print_verbose(self, s, thread_name=None): 
224              if self.verbose or self.full_verbose: 
225                  self.print_all(s, thread_name=thread_name) 
226   
227 -        def print_full_verbose(self, s): 
228              if self.full_verbose: 
229                  print("[ rosmake ] %s"%(s)) 
230   
231 -        def print_tail(self, s, tail_lines=40): 
232              lines = s.splitlines() 
233              if self.full_verbose: 
234                  tail_lines = len(lines) 
235   
236              num_lines = min(len(lines), tail_lines) 
237              if num_lines == tail_lines: 
238                  print("[ rosmake ] Last %d lines"%(num_lines)) 
239              else: 
240                  print("[ rosmake ] All %d lines"%(num_lines)) 
241              print("{" + "-"*79) 
242              for l in range(-num_lines, -1): 
243                  print("  %s"%(lines[l])) 
244              print("-"*79 + "}") 
245   
246 -        def _print_status(self, s): 
247              sys.stdout.write("%s\r"%(s)) 
248              sys.stdout.flush() 
249   
250          @staticmethod 
251 -        def terminal_width(): 
252              """Estimate the width of the terminal""" 
253              width = 0 
254              try: 
255                  import struct, fcntl, termios 
256                  s = struct.pack('HHHH', 0, 0, 0, 0) 
257                  x = fcntl.ioctl(1, termios.TIOCGWINSZ, s) 
258                  width = struct.unpack('HHHH', x)[1] 
259              except IOError: 
260                  pass 
261              if width <= 0: 
262                  try: 
263                      width = int(os.environ['COLUMNS']) 
264                  except: 
265                      pass 
266              if width <= 0: 
267                  width = 80 
268   
269              return width 
270   
271          @staticmethod 
272 -        def pad_str_to_width(str, width): 
273              """ Pad the string to be terminal width""" 
274              length = len(str) 
275              excess = 0 
276              if length < width: 
277                  excess = width - length 
278              return str + " "* excess 
279   
280   
281   
282 -class RosMakeAll: 
283 -    def __init__(self): 
284          self._result_lock = threading.Lock() 
285   
286          self.rospack = rospkg.RosPack() 
287          self.rosstack = rospkg.RosStack() 
288   
289          self.printer = Printer() 
290          self.result = {} 
291          self.paths = {} 
292          self.dependency_tracker = parallel_build.DependencyTracker(rospack=self.rospack) 
293          self.flag_tracker = package_stats.PackageFlagTracker(self.dependency_tracker) 
294          self.output = {} 
295          self.profile = {} 
296          self.ros_parallel_jobs = 0 
297          self.build_list = [] 
298          self.start_time = time.time() 
299          self.log_dir = "" 
300          self.logging_enabled = True 
301   
302 -    def num_packages_built(self): 
303          """ 
304          @return: number of packages that were built 
305          @rtype: int 
306          """ 
307          return len(list(self.result[argument].keys())) #py3k 
308   
309 -    def update_status(self, argument, start_times, right): 
310          self.printer.rosmake_cache_info(argument, start_times, right) 
311   
312 -    def build_or_recurse(self,p): 
313          if p in self.build_list: 
314              return 
315          for d in self.dependency_tracker.get_deps_1(p): 
316              self.build_or_recurse(d) 
317          try: # append it ot the list only if present 
318            self.rospack.get_path(p) 
319            self.build_list.append(p) 
320          except rospkg.ResourceNotFound as ex: 
321            if not self.robust_build: 
322              self.printer.print_all("Exiting due to missing package: %s"%ex) 
323              sys.exit(-1) 
324            else: 
325              self.printer.print_all("!"*20 + " Package %s does not exist. %s"%(p, ex) + "!"*20) 
326   
327   
328 -    def parallel_build_pkgs(self, build_queue, argument = None, threads = 1): 
329          self.profile[argument] = {} 
330          self.output[argument] = {} 
331          with self._result_lock: 
332              if argument not in self.result.keys(): 
333                  self.result[argument] = {} 
334   
335          cts = [] 
336          for i in range(0, threads): 
337            ct = parallel_build.CompileThread(str(i), build_queue, self, argument) 
338            #print "TTTH starting thread ", ct 
339            ct.start() 
340            cts.append(ct) 
341          for ct in cts: 
342            try: 
343              #print "TTTT Joining", ct 
344              ct.join() 
345              #print "TTTH naturally ended thread", ct 
346            except KeyboardInterrupt: 
347              self.printer.print_all( "TTTH Caught KeyboardInterrupt. Stopping build.") 
348              build_queue.stop() 
349              ct.join() 
350            except: #catch all 
351                self.printer.print_all("TTTH OTHER exception thrown!!!!!!!!!!!!!!!!!!!!!") 
352                ct.join() 
353          #print "All threads joined" 
354          all_pkgs_passed = True 
355          with self._result_lock: 
356              for v in self.result[argument].values(): 
357                  all_pkgs_passed = v and all_pkgs_passed 
358   
359          build_passed = build_queue.succeeded() and all_pkgs_passed 
360          return build_passed 
361   
362      # This function taken from 
363      # http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html 
364 -    def _subprocess_setup(self): 
365          # Python installs a SIGPIPE handler by default. This is usually not 
366          # what non-Python subprocesses expect. 
367          signal.signal(signal.SIGPIPE, signal.SIG_DFL) 
368   
369 -    def _build_package(self, package, argument=None): 
370          """ 
371          Lower-level routine for building a package. Handles execution of actual build command. 
372          @param package: package name 
373          @type  package: str 
374          """ 
375          local_env = os.environ.copy() 
376          if self.ros_parallel_jobs > 0: 
377              local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (self.ros_parallel_jobs, self.ros_parallel_jobs) 
378          elif "ROS_PARALLEL_JOBS" not in os.environ: #if no environment setup and no args fall back to # cpus 
379              # num_cpus check can (on OS X) trigger a Popen(), which has 
380              #the multithreading bug we wish to avoid on Py2.7. 
381              with _popen_lock: 
382                  num_cpus = parallel_build.num_cpus() 
383                  local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (num_cpus, num_cpus) 
384          local_env['SVN_CMDLINE'] = "svn --non-interactive" 
385          cmd = ["bash", "-c", "cd %s && %s "%(self.rospack.get_path(package), make_command()) ] #UNIXONLY 
386          if argument: 
387              cmd[-1] += argument 
388          self.printer.print_full_verbose (cmd) 
389          # #3883: make sure only one Popen command occurs at a time due to 
390          # http://bugs.python.org/issue13817 
391          with _popen_lock: 
392              command_line = subprocess.Popen(cmd, stdout=subprocess.PIPE,  stderr=subprocess.STDOUT, env=local_env, preexec_fn=self._subprocess_setup) 
393          (pstd_out, pstd_err) = command_line.communicate() # pstd_err should be None due to pipe above 
394          return (command_line.returncode, pstd_out) 
395   
396 -    def build(self, p, argument = None, robust_build=False): 
397          """ 
398          Build package 
399          @param p: package name 
400          @type  p: str 
401          """ 
402          return_string = "" 
403          try: 
404              # warn if ROS_BUILD_BLACKLIST encountered if applicable 
405              # do not build packages for which the build has failed 
406              if argument == "test":  # Tests are not build dependent 
407                  failed_packages = [] 
408              else: 
409                  with self._result_lock: 
410                      failed_packages = [j for j in self.result[argument] if not self.result[argument][j] == True] 
411   
412              (buildable, error, why) = self.flag_tracker.can_build(p, self.skip_blacklist, failed_packages) 
413              if buildable or self.robust_build: 
414                  start_time = time.time() 
415                  (returncode, pstd_out) = self._build_package(p, argument) 
416                  self.profile[argument][p] = time.time() - start_time 
417                  self.output[argument][p] = pstd_out 
418                  if argument: 
419                      log_type = "build_%s"%argument 
420                  else: 
421                      log_type = "build" 
422                  if not returncode: 
423                      self.printer.print_full_verbose( pstd_out) 
424                      with self._result_lock: 
425                          self.result[argument][p] = True 
426                      warnings = Warnings( pstd_out ) 
427                      num_warnings = len( warnings.warning_lines ) 
428                      if num_warnings > 0: 
429                          return_string =  "[PASS] [ %.2f seconds ] [ %d warnings "%(self.profile[argument][p], num_warnings) 
430                          warning_dict = warnings.analyze(); 
431                          for warntype,warnlines in warning_dict.iteritems(): 
432                              if len( warnlines ) > 0: 
433                                  return_string = return_string + '[ {0:d} {1} ] '.format(len(warnlines),warntype) 
434                          return_string = return_string + ' ]' 
435                      else: 
436                          return_string =  ("[PASS] [ %.2f seconds ]"%( self.profile[argument][p])) 
437                      self.output_to_file(p, log_type, pstd_out, num_warnings > 0) 
438                  else: 
439                      success = False 
440                      no_target = len(re.findall("No rule to make target", pstd_out)) > 0 
441                      interrupt = len(re.findall("Interrupt", pstd_out)) > 0 
442                      if no_target: 
443                          return_string = ( "[SKIP] No rule to make target %s"%( argument)) 
444                          success = True 
445                      elif interrupt: 
446                          return_string = ("[Interrupted]" ) 
447                      else: 
448                          return_string = ( "[FAIL] [ %.2f seconds ]"%( self.profile[argument][p])) 
449                      with self._result_lock: 
450                          self.result[argument][p] = True if no_target else False 
451   
452                      if success == False: #don't print tail if [SKIP] target 
453                          self.printer.print_tail( pstd_out) 
454                      self.output_to_file(p, log_type, pstd_out, always_print= not (no_target or interrupt)) 
455   
456                      return (success, return_string) 
457              else: 
458                  with self._result_lock: 
459                      self.result[argument][p] = error 
460   
461                  return_string += why 
462                  return(error, return_string) 
463              return (True, return_string) # this means that we didn't error in any case above 
464          except rospkg.ResourceNotFound as ex: 
465              with self._result_lock: 
466                  self.result[argument][p] = False 
467              self.printer.print_verbose ("[SKIP] Package %s not found\n" % p) 
468              self.output[argument][p] = "Package not found %s"%ex 
469              return (False, return_string) 
470               
471   
472 -    def output_to_file(self, package, log_type, stdout, always_print= False): 
473          if not self.logging_enabled: 
474              return 
475          package_log_dir = os.path.join(self.log_dir, package) 
476   
477          std_out_filename = os.path.join(package_log_dir, log_type + "_output.log") 
478          if not os.path.exists (package_log_dir): 
479              makedirs_with_parent_perms(package_log_dir) 
480          with open(std_out_filename, 'w') as stdout_file: 
481              stdout_file.write(stdout) 
482              print_string = "Output from build of package %s written to:\n[ rosmake ]    %s"%(package, std_out_filename) 
483              if always_print: 
484                  self.printer.print_all(print_string) 
485              else: 
486                  self.printer.print_full_verbose(print_string) 
487   
488 -    def generate_summary_output(self, log_dir): 
489          if not self.logging_enabled: 
490              return 
491   
492          self.printer.print_all("Results:") 
493          if 'clean' in self.result.keys(): 
494              self.printer.print_all("Cleaned %d packages."%len(self.result['clean'])) 
495          if None in self.result.keys(): 
496              build_failure_count = len([p for p in self.result[None].keys() if self.result[None][p] == False]) 
497              self.printer.print_all("Built %d packages with %d failures."%(len(self.result[None]), build_failure_count)) 
498          if 'test' in self.result.keys(): 
499              test_failure_count = len([p for p in self.result['test'].keys() if self.result['test'][p] == False]) 
500              self.printer.print_all("Tested %d packages with %d failures."%(len(self.result['test']), test_failure_count)) 
501          self.printer.print_all("Summary output to directory") 
502          self.printer.print_all("%s"%self.log_dir) 
503          if self.rejected_packages: 
504              self.printer.print_all("WARNING: Skipped command line arguments: %s because they could not be resolved to a stack name or a package name. "%self.rejected_packages) 
505   
506                              
507   
508          if None in self.result.keys(): 
509              if len(self.result[None].keys()) > 0: 
510                  buildfail_filename = os.path.join(log_dir, "buildfailures.txt") 
511                  with open(buildfail_filename, 'w') as bf: 
512                      bf.write("Build failures:\n") 
513                      for key in self.build_list: 
514                          if key in self.result[None].keys() and self.result[None][key] == False: 
515                              bf.write("%s\n"%key) 
516                  if  None in self.output.keys(): 
517                      buildfail_context_filename = os.path.join(log_dir, "buildfailures-with-context.txt") 
518                      with open(buildfail_context_filename, 'w') as bfwc: 
519                          bfwc.write("Build failures with context:\n") 
520                          for key in self.build_list: 
521                              if key in self.result[None].keys() and self.result[None][key] == False: 
522                                  bfwc.write("---------------------\n") 
523                                  bfwc.write("%s\n"%key) 
524                                  if key in self.output[None]: 
525                                      bfwc.write(self.output[None][key]) 
526   
527          if "test" in self.result.keys(): 
528              if len(self.result["test"].keys()) > 0: 
529                  testfail_filename = os.path.join(log_dir, "testfailures.txt") 
530                  with open(testfail_filename, 'w') as btwc: 
531                      btwc.write("Test failures:\n") 
532                      for key in self.build_list: 
533                          if key in self.result["test"].keys() and self.result["test"][key] == False: 
534                              btwc.write("%s\n"%key) 
535   
536                  if "test" in self.output.keys(): 
537                      testfail_filename = os.path.join(log_dir, "testfailures-with-context.txt") 
538                      with open(testfail_filename, 'w') as btwc: 
539                          btwc.write("Test failures with context:\n") 
540                          for key in self.build_list: 
541                              if key in self.result["test"].keys() and self.result["test"][key] == False: 
542                                  btwc.write("%s\n"%key) 
543                                  if key in self.output["test"]: 
544                                      btwc.write(self.output["test"][key]) 
545   
546          profile_filename = os.path.join(log_dir, "profile.txt") 
547          with open(profile_filename, 'w') as pf: 
548              pf.write(self.get_profile_string()) 
549                               
550                               
551   
552 -    def get_profile_string(self): 
553          output = '--------------\nProfile\n--------------\n' 
554          total = 0.0 
555          count = 1 
556          for key in self.build_list: 
557              build_results = ["[Not Built ]", "[  Built   ]", "[Build Fail]"]; 
558              test_results =  ["[Untested ]", "[Test Pass]", "[Test Fail]"]; 
559              build_result = 0 
560              test_result = 0 
561              test_time = 0.0 
562              build_time = 0.0 
563   
564              if None in self.result.keys(): 
565                  if key in self.result[None].keys(): 
566                      if self.result[None][key] == True: 
567                          build_result = 1 
568                      else: 
569                          build_result = 2 
570   
571              if "test" in self.profile.keys(): 
572                  if key in self.result["test"].keys(): 
573                      if self.result["test"][key] == True: 
574                          test_result = 1 
575                      else: 
576                          test_result = 2 
577   
578              if None in self.profile.keys(): 
579                  if key in self.profile[None].keys(): 
580                      build_time = self.profile[None][key] 
581   
582              if "test" in self.profile.keys(): 
583                  if key in self.profile["test"].keys(): 
584                      test_time = self.profile["test"][key] 
585                   
586                       
587              output = output + "%3d: %s in %.2f %s in %.2f --- %s\n"% (count, build_results[build_result], build_time , test_results[test_result], test_time, key) 
588              total = total + build_time 
589              count = count + 1 
590   
591          elapsed_time  =  self.finish_time - self.start_time 
592          output = output + "----------------\n" + "%.2f Cumulative,  %.2f Elapsed, %.2f Speedup \n"%(total, elapsed_time, float(total) / float(elapsed_time)) 
593          return output 
594   
595 -    def main(self): 
596          """ 
597          main command-line entrypoint 
598          """ 
599          parser = OptionParser(usage="usage: %prog [options] [PACKAGE]...", 
600                                description="rosmake recursively builds all dependencies before building a package", prog='rosmake') 
601          parser.add_option("--test-only", dest="test_only", default=False, 
602                            action="store_true", help="only run tests") 
603          parser.add_option("-t", dest="test", default=False, 
604                            action="store_true", help="build and test packages") 
605          parser.add_option("-a", "--all", dest="build_all", default=False, 
606                            action="store_true", help="select all packages") 
607          parser.add_option("-i", "--mark-installed", dest="mark_installed", default=False, 
608                            action="store_true", help="On successful build, mark specified packages as installed with ROS_NOBUILD") 
609          parser.add_option("-u", "--unmark-installed", dest="unmark_installed", default=False, 
610                            action="store_true", help="Remove ROS_NOBUILD from the specified packages.  This will not build anything.") 
611          parser.add_option("-v", dest="verbose", default=False, 
612                            action="store_true", help="display errored builds") 
613          parser.add_option("-r","-k", "--robust", dest="best_effort", default=False, 
614                             action="store_true", help="do not stop build on error") 
615          parser.add_option("--build-everything", dest="robust", default=False, 
616                             action="store_true", help="build all packages regardless of errors") 
617          parser.add_option("-V", dest="full_verbose", default=False, 
618                            action="store_true", help="display all builds") 
619          parser.add_option("-s", "--specified-only", dest="specified_only", default=False, 
620                            action="store_true", help="only build packages specified on the command line") 
621          parser.add_option("--buildtest", dest="buildtest", 
622                            action="append", help="package to buildtest") 
623          parser.add_option("--buildtest1", dest="buildtest1", 
624                            action="append", help="package to buildtest1") 
625          parser.add_option("--output", dest="output_dir", 
626                            action="store", help="where to output results") 
627          parser.add_option("--pre-clean", dest="pre_clean", 
628                            action="store_true", help="run make clean first") 
629          parser.add_option("--bootstrap", dest="bootstrap", default=False, 
630                            action="store_true", help="DEPRECATED, UNUSED") 
631          parser.add_option("--disable-logging", dest="logging_enabled", default=True, 
632                            action="store_false", help="turn off all logs") 
633          parser.add_option("--target", dest="target", 
634                            action="store", help="run make with this target") 
635          parser.add_option("--pjobs", dest="ros_parallel_jobs", type="int", 
636                            action="store", help="Override ROS_PARALLEL_JOBS environment variable with this number of jobs.") 
637          parser.add_option("--threads", dest="threads", type="int", default = os.environ.get("ROSMAKE_THREADS", parallel_build.num_cpus()), 
638                            action="store", help="Build up to N packages in parallel") 
639          parser.add_option("--profile", dest="print_profile", default=False, 
640                            action="store_true", help="print time profile after build") 
641          parser.add_option("--skip-blacklist", dest="skip_blacklist",  
642                            default=False, action="store_true",  
643                            help="skip packages containing a file called ROS_BUILD_BLACKLIST (Default behavior will ignore the presence of ROS_BUILD_BLACKLIST)") 
644          parser.add_option("--skip-blacklist-osx", dest="skip_blacklist_osx",  
645                            default=False, action="store_true",  
646                            help="deprecated option. it will do nothing, please use platform declarations and --require-platform instead") 
647   
648          parser.add_option("--status-rate", dest="status_update_rate", 
649                            action="store", help="How fast to update the status bar in Hz.  Default: 5Hz") 
650           
651   
652          options, args = parser.parse_args() 
653          self.printer.print_all('rosmake starting...') 
654   
655          rospack = self.rospack 
656          rosstack = self.rosstack 
657   
658          testing = False 
659          building = True 
660          if options.test_only: 
661              testing = True 
662              building = False 
663          elif options.test: 
664              testing = True 
665   
666          if options.ros_parallel_jobs: 
667              self.ros_parallel_jobs = options.ros_parallel_jobs 
668   
669          self.robust_build = options.robust 
670          self.best_effort = options.best_effort 
671          self.threads = options.threads 
672          self.skip_blacklist = options.skip_blacklist 
673          if options.skip_blacklist_osx: 
674              self.printer.print_all("Option --skip-blacklist-osx is deprecated. It will do nothing, please use platform declarations and --require-platform instead"); 
675          self.logging_enabled = options.logging_enabled 
676   
677          # pass through verbosity options 
678          self.printer.full_verbose = options.full_verbose 
679          self.printer.verbose = options.verbose 
680          if options.status_update_rate: 
681              if float(options.status_update_rate)> 0: 
682                  self.printer.duration = 1.0/float(options.status_update_rate) 
683              else: 
684                  self.printer.duration = 0 
685   
686          packages = [] 
687          #load packages from arguments 
688          if options.build_all: 
689              packages = [x for x in rospack.list() if not self.rospack.get_manifest(x).is_catkin] 
690              self.printer.print_all( "Building all packages") 
691          else:      # no need to extend if all already selected    
692              if options.buildtest: 
693                for p in options.buildtest: 
694                  packages.extend(self.rospack.get_depends_on(p))  
695                  self.printer.print_all( "buildtest requested for package %s adding it and all dependent packages: "%p) 
696   
697              if options.buildtest1: 
698                for p in options.buildtest1: 
699                  packages.extend(self.rospack.get_depends_on(p, implicit=False))  
700                  self.printer.print_all( "buildtest1 requested for package %s adding it and all depends-on1 packages: "%p) 
701   
702          if len(packages) == 0 and len(args) == 0: 
703              p = os.path.basename(os.path.abspath('.')) 
704              try: 
705                if os.path.samefile(rospack.get_path(p), '.'): 
706                  packages = [p] 
707                  self.printer.print_all( "No package specified.  Building %s"%packages) 
708                else: 
709                  self.printer.print_all("No package selected and the current directory is not the correct path for package '%s'."%p) 
710                   
711              except rospkg.ResourceNotFound as ex: 
712                  try: 
713                      stack_dir = rosstack.get_path(p) 
714                      if os.path.samefile(stack_dir, '.'): 
715                          packages = [p] 
716                          self.printer.print_all( "No package specified.  Building stack %s"%packages) 
717                      else: 
718                          self.printer.print_all("No package or stack arguments and the current directory is not the correct path for stack '%s'. Stack directory is: %s."%(p, rosstack.get_path(p))) 
719                  except: 
720                      self.printer.print_all("No package or stack specified.  And current directory '%s' is not a package name or stack name."%p) 
721          else: 
722              packages.extend(args) 
723   
724          self.printer.print_all( "Packages requested are: %s"%packages) 
725           
726          # Setup logging 
727          if self.logging_enabled: 
728            date_time_stamp =  "rosmake_output-" + time.strftime("%Y%m%d-%H%M%S") 
729            if options.output_dir: 
730                #self.log_dir = os.path.join(os.getcwd(), options.output_dir, date_time_stamp); 
731                self.log_dir = os.path.abspath(options.output_dir) 
732            else: 
733                self.log_dir = os.path.join(rospkg.get_ros_home(), "rosmake", date_time_stamp); 
734   
735            self.printer.print_all("Logging to directory %s"%self.log_dir) 
736            if os.path.exists (self.log_dir) and not os.path.isdir(self.log_dir): 
737                self.printer.print_all( "Log destination %s is a file; please remove it or choose a new destination"%self.log_dir) 
738                sys.exit(1) 
739            if not os.path.exists (self.log_dir): 
740                self.printer.print_verbose("%s doesn't exist: creating"%self.log_dir) 
741                makedirs_with_parent_perms(self.log_dir) 
742   
743            self.printer.print_verbose("Finished setting up logging") 
744   
745          stacks_arguments = [s for s in packages if s in rosstack.list()] 
746          (self.specified_packages, self.rejected_packages) = rospkg.expand_to_packages(packages, rospack, rosstack) 
747   
748          self.printer.print_all("Expanded args %s to:\n%s"%(packages, self.specified_packages)) 
749          if self.rejected_packages: 
750              self.printer.print_all("WARNING: The following args could not be parsed as stacks or packages: %s"%self.rejected_packages) 
751          if len(self.specified_packages) + len(stacks_arguments) == 0: 
752              self.printer.print_all("ERROR: No arguments could be parsed into valid package or stack names.") 
753              self.printer.running = False 
754              return False 
755   
756          if options.unmark_installed: 
757              for p in self.specified_packages: 
758                  if self.flag_tracker.remove_nobuild(p): 
759                      self.printer.print_all("Removed ROS_NOBUILD from %s"%p) 
760              self.printer.running = False 
761              return True 
762               
763          required_packages = self.specified_packages[:] 
764   
765          # catch packages of dependent stacks when specified stack is zero-sized #3528 
766          # add them to required list but not the specified list.  
767          for s in stacks_arguments: 
768              if not rosstack.packages_of(s): 
769                  for d in rosstack.get_depends(s, implicit=False): 
770                      try: 
771                          required_packages.extend(rosstack.packages_of(d)) 
772                      except ResourceNotFound: 
773                          self.printer.print_all('WARNING: The stack "%s" was not found. We will assume it is using the new buildsystem and try to continue...' % d) 
774   
775          # deduplicate required_packages 
776          required_packages = list(set(required_packages)) 
777   
778          # make sure all dependencies are satisfied and if not warn 
779          buildable_packages = [] 
780          for p in required_packages: 
781              (buildable, error, str) = self.flag_tracker.can_build(p, self.skip_blacklist, [], False) 
782              if buildable:  
783                  buildable_packages.append(p) 
784   
785          #generate the list of packages necessary to build(in order of dependencies) 
786          counter = 0 
787          for p in required_packages: 
788   
789              counter = counter + 1 
790              self.printer.print_verbose( "Processing %s and all dependencies(%d of %d requested)"%(p, counter, len(packages))) 
791              self.build_or_recurse(p) 
792   
793          # remove extra packages if specified-only flag is set 
794          if options.specified_only: 
795            new_list = [] 
796            for pkg in self.build_list: 
797              if pkg in self.specified_packages: 
798                new_list.append(pkg) 
799                self.dependency_tracker = parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack) # this will make the tracker only respond to packages in the list 
800           
801            self.printer.print_all("specified-only option was used, only building packages %s"%new_list) 
802            self.build_list = new_list 
803   
804          if options.pre_clean: 
805            build_queue = parallel_build.BuildQueue(self.build_list, parallel_build.DependencyTracker([], rospack=self.rospack), robust_build = True) 
806            self.parallel_build_pkgs(build_queue, "clean", threads = options.threads) 
807   
808          build_passed = True 
809   
810          if building: 
811            self.printer.print_verbose ("Building packages %s"% self.build_list) 
812            build_queue = parallel_build.BuildQueue(self.build_list, self.dependency_tracker, robust_build = options.robust or options.best_effort) 
813            if None not in self.result.keys(): 
814                  self.result[None] = {} 
815   
816            build_passed = self.parallel_build_pkgs(build_queue, options.target, threads = options.threads) 
817   
818          tests_passed = True 
819          if build_passed and testing: 
820              self.printer.print_verbose ("Testing packages %s"% packages) 
821              build_queue = parallel_build.BuildQueue(self.specified_packages, parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack), robust_build = True) 
822              tests_passed = self.parallel_build_pkgs(build_queue, "test", threads = 1) 
823   
824   
825          if  options.mark_installed: 
826              if build_passed and tests_passed:  
827                  for p in self.specified_packages: 
828                      if self.flag_tracker.add_nobuild(p): 
829                          self.printer.print_all("Marking %s as installed with a ROS_NOBUILD file"%p) 
830              else: 
831                  self.printer.print_all("All builds and tests did not pass cannot mark packages as installed. ") 
832   
833   
834          self.finish_time = time.time() #note: before profiling 
835          self.generate_summary_output(self.log_dir) 
836           
837          if options.print_profile: 
838              self.printer.print_all (self.get_profile_string()) 
839   
840          self.printer.running = False 
841          return build_passed and tests_passed 
842