Source Code for Module rosmake.engine

  1  #! /usr/bin/env python 
  2   
  3  # Copyright (c) 2009, Willow Garage, Inc. 
  4  # All rights reserved. 
  5  #  
  6  # Redistribution and use in source and binary forms, with or without 
  7  # modification, are permitted provided that the following conditions are met: 
  8  #  
  9  #     * Redistributions of source code must retain the above copyright 
 10  #       notice, this list of conditions and the following disclaimer. 
 11  #     * Redistributions in binary form must reproduce the above copyright 
 12  #       notice, this list of conditions and the following disclaimer in the 
 13  #       documentation and/or other materials provided with the distribution. 
 14  #     * Neither the name of Willow Garage, Inc. nor the names of its 
 15  #       contributors may be used to endorse or promote products derived from 
 16  #       this software without specific prior written permission. 
 17  #  
 18  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
 19  # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
 20  # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
 21  # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
 22  # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
 23  # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
 24  # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
 25  # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
 26  # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
 27  # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
 28  # POSSIBILITY OF SUCH DAMAGE. 
 29   
 30  # Author Tully Foote/tfoote@willowgarage.com 
 31   
 32  from __future__ import print_function 
 33   
 34  import os 
 35  import re 
 36  import signal 
 37  import sys 
 38  import subprocess 
 39  import time 
 40  import threading 
 41  import traceback 
 42   
 43  import rospkg 
 44  from rospkg import ResourceNotFound 
 45   
 46  try:  
 47      from exceptions import SystemExit #Python 2.x 
 48  except ImportError:  
 49      pass #Python 3.x (in Python 3, 'exceptions' is always imported)  
 50   
 51  from operator import itemgetter 
 52   
 53  from . import parallel_build 
 54  from . import package_stats 
 55   
 56  from optparse import OptionParser 
 57  from .gcc_output_parse import Warnings 
 58   
 59  # #3883 
 60  _popen_lock = threading.Lock() 
 61   
 62 -def make_command(): 
 63      """ 
 64      @return: name of 'make' command 
 65      @rtype: str 
 66      """ 
 67      return os.environ.get("MAKE", "make") 
 68   
 69  # this is a copy of the roslogging utility. it's been moved here as it is a common 
 70  # routine for programs using accessing ROS directories 
 71 -def makedirs_with_parent_perms(p): 
 72      """ 
 73      Create the directory using the permissions of the nearest 
 74      (existing) parent directory. This is useful for logging, where a 
 75      root process sometimes has to log in the user's space. 
 76      @param p: directory to create 
 77      @type  p: str 
 78      """     
 79      p = os.path.abspath(p) 
 80      parent = os.path.dirname(p) 
 81      # recurse upwards, checking to make sure we haven't reached the 
 82      # top 
 83      if not os.path.exists(p) and p and parent != p: 
 84          makedirs_with_parent_perms(parent) 
 85          s = os.stat(parent) 
 86          os.mkdir(p) 
 87   
 88          # if perms of new dir don't match, set anew 
 89          s2 = os.stat(p) 
 90          if s.st_uid != s2.st_uid or s.st_gid != s2.st_gid: 
 91              os.chown(p, s.st_uid, s.st_gid) 
 92          if s.st_mode != s2.st_mode: 
 93              os.chmod(p, s.st_mode)     
 94   
 95 -class Printer: 
 96     # storage for the instance reference 
 97      __instance = None 
 98   
 99 -    def __init__(self): 
100          """ Create singleton instance """ 
101          # Check whether we already have an instance 
102          if Printer.__instance is None: 
103              # Create and remember instance 
104              Printer.__instance = Printer.__impl() 
105   
106          # Store instance reference as the only member in the handle 
107          self.__dict__['_Printer__instance'] = Printer.__instance 
108   
109 -    def __getattr__(self, attr): 
110          """ Delegate access to implementation """ 
111          return getattr(self.__instance, attr) 
112   
113 -    def __setattr__(self, attr, value): 
114          """ Delegate access to implementation """ 
115          return setattr(self.__instance, attr, value) 
116       
117 -    def __enter__(self): 
118          """Pass through for the __enter__ function for the __instance""" 
119          return self.__instance.__enter__() 
120       
121 -    def __exit__(self, mtype, value, tb): 
122          """Pass through for the __exit__ function for the __instance""" 
123          return self.__instance.__exit__(mtype, value, tb) 
124       
125 -    class __impl(threading.Thread): 
126 -        def __init__(self): 
127              threading.Thread.__init__(self) 
128              self.build_queue = None 
129              self.condition = threading.Condition() 
130              self.running = True 
131              self.done = False 
132              self.status = "" 
133              self.verbose = False 
134              self.full_verbose = False 
135              self.duration = 1./10. 
136              self._last_status = None 
137   
138              # Rosmake specific data 
139              self.cache_argument = None 
140              self.cache_right = '' 
141              self.pkg_start_times = {} 
142   
143 -        def shutdown(self): 
144              self.running = False 
145              cycles = 10 
146              for i in range(0,cycles):# sleep for at least 2 cycles of the status testing 'cycles' times 
147                  if self.done: 
148                      #print "SUCCESSFULLY SHUTDOWN" 
149                      return True 
150                  #print "Sleeping for %f FOR SHUTDOWN.  %d threads running"%(max(self.duration/cycles*2, 0.01), threading.activeCount()) 
151                  time.sleep(max(self.duration, 0.1)/cycles*2)  
152              raise Exception("Failed to shutdown status thread in %.2f seconds"%(self.duration * 2)) 
153   
154 -        def __enter__(self): 
155              self.start() 
156 -        def __exit__(self, mtype, value, tb): 
157              self.shutdown() 
158              if value: 
159                  if not mtype == type(SystemExit()): 
160                      traceback.print_exception(mtype, value, tb) 
161                  else: 
162                      sys.exit(value) 
163   
164 -        def run(self): 
165              while self.running: 
166                  #shutdown if duration set to zero 
167                  if self.duration <= 0: 
168                      self.running = False 
169                      break 
170                  self.set_status_from_cache() 
171                  if len(self.pkg_start_times.keys()) > 0: 
172                      n = self.terminal_width() - len(self.status) 
173                      status = self.status 
174                      if n > 0: 
175                          status = " "*n + self.status 
176                      if status != self._last_status: 
177                          self._print_status("%s"%status) 
178                          self._last_status = status 
179                  time.sleep(self.duration)  
180              self.done = True 
181              #print "STATUS THREAD FINISHED" 
182   
183 -        def rosmake_cache_info(self, argument, start_times, right): 
184              self.cache_argument = argument 
185              self.pkg_start_times = start_times 
186              self.cache_right = right 
187   
188 -        def rosmake_pkg_times_to_string(self, start_times): 
189              threads = [] 
190              for p, t in sorted(start_times.items(), key=itemgetter(1)): #py3k 
191                  threads.append("[ %s: %.1f sec ]"%(p, time.time() - t)) 
192   
193              return " ".join(threads) 
194   
195 -        def set_status_from_cache(self): 
196              if self.cache_argument: 
197                  self.set_status("[ make %s ] "%self.cache_argument + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
198              else: 
199                  self.set_status("[ make ] " + self.rosmake_pkg_times_to_string(self.pkg_start_times), self.cache_right) 
200   
201 -        def set_status(self, left, right = ''): 
202              header = "[ rosmake ] " 
203              h = len(header) 
204              l = len(left) 
205              r = len(right) 
206              w = self.terminal_width() 
207              if l + r < w - h: 
208                  padding = w - h - l - r 
209                  self.status = header + left + " "*padding + right 
210              else: 
211                  self.status = header + left[:(w - h - r - 4)] + "... " + right 
212   
213 -        def print_all(self, s, thread_name=None): 
214              if thread_name is None: 
215                  str = "[ rosmake ] %s"%s 
216                   
217   
218              else: 
219                  str = "[rosmake-%s] %s"%(thread_name, s) 
220              sys.stdout.write(self.pad_str_to_width(str, self.terminal_width())+"\n") 
221              sys.stdout.flush() 
222   
223 -        def print_verbose(self, s, thread_name=None): 
224              if self.verbose or self.full_verbose: 
225                  self.print_all(s, thread_name=thread_name) 
226   
227 -        def print_full_verbose(self, s): 
228              if self.full_verbose: 
229                  print("[ rosmake ] %s"%(s)) 
230   
231 -        def print_tail(self, s, tail_lines=40): 
232              lines = s.splitlines() 
233              if self.full_verbose: 
234                  tail_lines = len(lines) 
235   
236              num_lines = min(len(lines), tail_lines) 
237              if num_lines == tail_lines: 
238                  print("[ rosmake ] Last %d lines"%(num_lines)) 
239              else: 
240                  print("[ rosmake ] All %d lines"%(num_lines)) 
241              print("{" + "-"*79) 
242              for l in range(-num_lines, -1): 
243                  print("  %s"%(lines[l])) 
244              print("-"*79 + "}") 
245   
246 -        def _print_status(self, s): 
247              sys.stdout.write("%s\r"%(s)) 
248              sys.stdout.flush() 
249   
250          @staticmethod 
251 -        def terminal_width(): 
252              """Estimate the width of the terminal""" 
253              width = 0 
254              try: 
255                  import struct, fcntl, termios 
256                  s = struct.pack('HHHH', 0, 0, 0, 0) 
257                  x = fcntl.ioctl(1, termios.TIOCGWINSZ, s) 
258                  width = struct.unpack('HHHH', x)[1] 
259              except IOError: 
260                  pass 
261              if width <= 0: 
262                  try: 
263                      width = int(os.environ['COLUMNS']) 
264                  except: 
265                      pass 
266              if width <= 0: 
267                  width = 80 
268   
269              return width 
270   
271          @staticmethod 
272 -        def pad_str_to_width(str, width): 
273              """ Pad the string to be terminal width""" 
274              length = len(str) 
275              excess = 0 
276              if length < width: 
277                  excess = width - length 
278              return str + " "* excess 
279   
280   
281   
282 -class RosMakeAll: 
283 -    def __init__(self): 
284          self._result_lock = threading.Lock() 
285   
286          self.rospack = rospkg.RosPack() 
287          self.rosstack = rospkg.RosStack() 
288   
289          self.printer = Printer() 
290          self.result = {} 
291          self.paths = {} 
292          self.dependency_tracker = parallel_build.DependencyTracker(rospack=self.rospack) 
293          self.flag_tracker = package_stats.PackageFlagTracker(self.dependency_tracker) 
294          self.output = {} 
295          self.profile = {} 
296          self.ros_parallel_jobs = 0 
297          self.build_list = [] 
298          self.start_time = time.time() 
299          self.log_dir = "" 
300          self.logging_enabled = True 
301   
302 -    def num_packages_built(self): 
303          """ 
304          @return: number of packages that were built 
305          @rtype: int 
306          """ 
307          return len(list(self.result[argument].keys())) #py3k 
308   
309 -    def update_status(self, argument, start_times, right): 
310          self.printer.rosmake_cache_info(argument, start_times, right) 
311   
312 -    def build_or_recurse(self,p): 
313          if p in self.build_list: 
314              return 
315          for d in self.dependency_tracker.get_deps_1(p): 
316              self.build_or_recurse(d) 
317          try: # append it ot the list only if present 
318            self.rospack.get_path(p) 
319            self.build_list.append(p) 
320          except rospkg.ResourceNotFound as ex: 
321            if not self.robust_build: 
322              self.printer.print_all("Exiting due to missing package: %s"%ex) 
323              sys.exit(-1) 
324            else: 
325              self.printer.print_all("!"*20 + " Package %s does not exist. %s"%(p, ex) + "!"*20) 
326   
327   
328 -    def parallel_build_pkgs(self, build_queue, argument = None, threads = 1): 
329          self.profile[argument] = {} 
330          self.output[argument] = {} 
331          with self._result_lock: 
332              if argument not in self.result.keys(): 
333                  self.result[argument] = {} 
334   
335          cts = [] 
336          for i in range(0, threads): 
337            ct = parallel_build.CompileThread(str(i), build_queue, self, argument) 
338            #print "TTTH starting thread ", ct 
339            ct.start() 
340            cts.append(ct) 
341          for ct in cts: 
342            try: 
343              #print "TTTT Joining", ct 
344              ct.join() 
345              #print "TTTH naturally ended thread", ct 
346            except KeyboardInterrupt: 
347              self.printer.print_all( "TTTH Caught KeyboardInterrupt. Stopping build.") 
348              build_queue.stop() 
349              ct.join() 
350            except: #catch all 
351                self.printer.print_all("TTTH OTHER exception thrown!!!!!!!!!!!!!!!!!!!!!") 
352                ct.join() 
353          #print "All threads joined" 
354          all_pkgs_passed = True 
355          with self._result_lock: 
356              for v in self.result[argument].values(): 
357                  all_pkgs_passed = v and all_pkgs_passed 
358   
359          build_passed = build_queue.succeeded() and all_pkgs_passed 
360          return build_passed 
361   
362      # This function taken from 
363      # http://www.chiark.greenend.org.uk/ucgi/~cjwatson/blosxom/2009-07-02-python-sigpipe.html 
364 -    def _subprocess_setup(self): 
365          # Python installs a SIGPIPE handler by default. This is usually not 
366          # what non-Python subprocesses expect. 
367          signal.signal(signal.SIGPIPE, signal.SIG_DFL) 
368   
369 -    def _build_package(self, package, argument=None): 
370          """ 
371          Lower-level routine for building a package. Handles execution of actual build command. 
372          @param package: package name 
373          @type  package: str 
374          """ 
375          local_env = os.environ.copy() 
376          if self.ros_parallel_jobs > 0: 
377              local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (self.ros_parallel_jobs, self.ros_parallel_jobs) 
378          elif "ROS_PARALLEL_JOBS" not in os.environ: #if no environment setup and no args fall back to # cpus 
379              # num_cpus check can (on OS X) trigger a Popen(), which has 
380              #the multithreading bug we wish to avoid on Py2.7. 
381              with _popen_lock: 
382                  num_cpus = parallel_build.num_cpus() 
383                  local_env['ROS_PARALLEL_JOBS'] = "-j%d -l%d" % (num_cpus, num_cpus) 
384          local_env['SVN_CMDLINE'] = "svn --non-interactive" 
385          cmd = ["bash", "-c", "cd %s && %s "%(self.rospack.get_path(package), make_command()) ] #UNIXONLY 
386          if argument: 
387              cmd[-1] += argument 
388          self.printer.print_full_verbose (cmd) 
389          # #3883: make sure only one Popen command occurs at a time due to 
390          # http://bugs.python.org/issue13817 
391          with _popen_lock: 
392              command_line = subprocess.Popen(cmd, stdout=subprocess.PIPE,  stderr=subprocess.STDOUT, env=local_env, preexec_fn=self._subprocess_setup) 
393          (pstd_out, pstd_err) = command_line.communicate() # pstd_err should be None due to pipe above 
394          if not isinstance(pstd_out, str): 
395              pstd_out = pstd_out.decode() 
396          return (command_line.returncode, pstd_out) 
397   
398 -    def build(self, p, argument = None, robust_build=False): 
399          """ 
400          Build package 
401          @param p: package name 
402          @type  p: str 
403          """ 
404          return_string = "" 
405          try: 
406              # warn if ROS_BUILD_BLACKLIST encountered if applicable 
407              # do not build packages for which the build has failed 
408              if argument == "test":  # Tests are not build dependent 
409                  failed_packages = [] 
410              else: 
411                  with self._result_lock: 
412                      failed_packages = [j for j in self.result[argument] if not self.result[argument][j] == True] 
413   
414              (buildable, error, why) = self.flag_tracker.can_build(p, self.skip_blacklist, failed_packages) 
415              if buildable or self.robust_build: 
416                  start_time = time.time() 
417                  (returncode, pstd_out) = self._build_package(p, argument) 
418                  self.profile[argument][p] = time.time() - start_time 
419                  self.output[argument][p] = pstd_out 
420                  if argument: 
421                      log_type = "build_%s"%argument 
422                  else: 
423                      log_type = "build" 
424                  if not returncode: 
425                      self.printer.print_full_verbose( pstd_out) 
426                      with self._result_lock: 
427                          self.result[argument][p] = True 
428                      warnings = Warnings( pstd_out ) 
429                      num_warnings = len( warnings.warning_lines ) 
430                      if num_warnings > 0: 
431                          return_string =  "[PASS] [ %.2f seconds ] [ %d warnings "%(self.profile[argument][p], num_warnings) 
432                          warning_dict = warnings.analyze(); 
433                          for warntype,warnlines in warning_dict.items(): 
434                              if len( warnlines ) > 0: 
435                                  return_string = return_string + '[ {0:d} {1} ] '.format(len(warnlines),warntype) 
436                          return_string = return_string + ' ]' 
437                      else: 
438                          return_string =  ("[PASS] [ %.2f seconds ]"%( self.profile[argument][p])) 
439                      self.output_to_file(p, log_type, pstd_out, num_warnings > 0) 
440                  else: 
441                      success = False 
442                      no_target = len(re.findall("No rule to make target", pstd_out)) > 0 
443                      interrupt = len(re.findall("Interrupt", pstd_out)) > 0 
444                      if no_target: 
445                          return_string = ( "[SKIP] No rule to make target %s"%( argument)) 
446                          success = True 
447                      elif interrupt: 
448                          return_string = ("[Interrupted]" ) 
449                      else: 
450                          return_string = ( "[FAIL] [ %.2f seconds ]"%( self.profile[argument][p])) 
451                      with self._result_lock: 
452                          self.result[argument][p] = True if no_target else False 
453   
454                      if success == False: #don't print tail if [SKIP] target 
455                          self.printer.print_tail( pstd_out) 
456                      self.output_to_file(p, log_type, pstd_out, always_print= not (no_target or interrupt)) 
457   
458                      return (success, return_string) 
459              else: 
460                  with self._result_lock: 
461                      self.result[argument][p] = error 
462   
463                  return_string += why 
464                  return(error, return_string) 
465              return (True, return_string) # this means that we didn't error in any case above 
466          except rospkg.ResourceNotFound as ex: 
467              with self._result_lock: 
468                  self.result[argument][p] = False 
469              self.printer.print_verbose ("[SKIP] Package %s not found\n" % p) 
470              self.output[argument][p] = "Package not found %s"%ex 
471              return (False, return_string) 
472               
473   
474 -    def output_to_file(self, package, log_type, stdout, always_print= False): 
475          if not self.logging_enabled: 
476              return 
477          package_log_dir = os.path.join(self.log_dir, package) 
478   
479          std_out_filename = os.path.join(package_log_dir, log_type + "_output.log") 
480          if not os.path.exists (package_log_dir): 
481              makedirs_with_parent_perms(package_log_dir) 
482          with open(std_out_filename, 'w') as stdout_file: 
483              stdout_file.write(stdout) 
484              print_string = "Output from build of package %s written to:\n[ rosmake ]    %s"%(package, std_out_filename) 
485              if always_print: 
486                  self.printer.print_all(print_string) 
487              else: 
488                  self.printer.print_full_verbose(print_string) 
489   
490 -    def generate_summary_output(self, log_dir): 
491          if not self.logging_enabled: 
492              return 
493   
494          self.printer.print_all("Results:") 
495          if 'clean' in self.result.keys(): 
496              self.printer.print_all("Cleaned %d packages."%len(self.result['clean'])) 
497          if None in self.result.keys(): 
498              build_failure_count = len([p for p in self.result[None].keys() if self.result[None][p] == False]) 
499              self.printer.print_all("Built %d packages with %d failures."%(len(self.result[None]), build_failure_count)) 
500          if 'test' in self.result.keys(): 
501              test_failure_count = len([p for p in self.result['test'].keys() if self.result['test'][p] == False]) 
502              self.printer.print_all("Tested %d packages with %d failures."%(len(self.result['test']), test_failure_count)) 
503          self.printer.print_all("Summary output to directory") 
504          self.printer.print_all("%s"%self.log_dir) 
505          if self.rejected_packages: 
506              self.printer.print_all("WARNING: Skipped command line arguments: %s because they could not be resolved to a stack name or a package name. "%self.rejected_packages) 
507   
508                              
509   
510          if None in self.result.keys(): 
511              if len(self.result[None].keys()) > 0: 
512                  buildfail_filename = os.path.join(log_dir, "buildfailures.txt") 
513                  with open(buildfail_filename, 'w') as bf: 
514                      bf.write("Build failures:\n") 
515                      for key in self.build_list: 
516                          if key in self.result[None].keys() and self.result[None][key] == False: 
517                              bf.write("%s\n"%key) 
518                  if  None in self.output.keys(): 
519                      buildfail_context_filename = os.path.join(log_dir, "buildfailures-with-context.txt") 
520                      with open(buildfail_context_filename, 'w') as bfwc: 
521                          bfwc.write("Build failures with context:\n") 
522                          for key in self.build_list: 
523                              if key in self.result[None].keys() and self.result[None][key] == False: 
524                                  bfwc.write("---------------------\n") 
525                                  bfwc.write("%s\n"%key) 
526                                  if key in self.output[None]: 
527                                      bfwc.write(self.output[None][key]) 
528   
529          if "test" in self.result.keys(): 
530              if len(self.result["test"].keys()) > 0: 
531                  testfail_filename = os.path.join(log_dir, "testfailures.txt") 
532                  with open(testfail_filename, 'w') as btwc: 
533                      btwc.write("Test failures:\n") 
534                      for key in self.build_list: 
535                          if key in self.result["test"].keys() and self.result["test"][key] == False: 
536                              btwc.write("%s\n"%key) 
537   
538                  if "test" in self.output.keys(): 
539                      testfail_filename = os.path.join(log_dir, "testfailures-with-context.txt") 
540                      with open(testfail_filename, 'w') as btwc: 
541                          btwc.write("Test failures with context:\n") 
542                          for key in self.build_list: 
543                              if key in self.result["test"].keys() and self.result["test"][key] == False: 
544                                  btwc.write("%s\n"%key) 
545                                  if key in self.output["test"]: 
546                                      btwc.write(self.output["test"][key]) 
547   
548          profile_filename = os.path.join(log_dir, "profile.txt") 
549          with open(profile_filename, 'w') as pf: 
550              pf.write(self.get_profile_string()) 
551                               
552                               
553   
554 -    def get_profile_string(self): 
555          output = '--------------\nProfile\n--------------\n' 
556          total = 0.0 
557          count = 1 
558          for key in self.build_list: 
559              build_results = ["[Not Built ]", "[  Built   ]", "[Build Fail]"]; 
560              test_results =  ["[Untested ]", "[Test Pass]", "[Test Fail]"]; 
561              build_result = 0 
562              test_result = 0 
563              test_time = 0.0 
564              build_time = 0.0 
565   
566              if None in self.result.keys(): 
567                  if key in self.result[None].keys(): 
568                      if self.result[None][key] == True: 
569                          build_result = 1 
570                      else: 
571                          build_result = 2 
572   
573              if "test" in self.profile.keys(): 
574                  if key in self.result["test"].keys(): 
575                      if self.result["test"][key] == True: 
576                          test_result = 1 
577                      else: 
578                          test_result = 2 
579   
580              if None in self.profile.keys(): 
581                  if key in self.profile[None].keys(): 
582                      build_time = self.profile[None][key] 
583   
584              if "test" in self.profile.keys(): 
585                  if key in self.profile["test"].keys(): 
586                      test_time = self.profile["test"][key] 
587                   
588                       
589              output = output + "%3d: %s in %.2f %s in %.2f --- %s\n"% (count, build_results[build_result], build_time , test_results[test_result], test_time, key) 
590              total = total + build_time 
591              count = count + 1 
592   
593          elapsed_time  =  self.finish_time - self.start_time 
594          output = output + "----------------\n" + "%.2f Cumulative,  %.2f Elapsed, %.2f Speedup \n"%(total, elapsed_time, float(total) / float(elapsed_time)) 
595          return output 
596   
597 -    def main(self): 
598          """ 
599          main command-line entrypoint 
600          """ 
601          parser = OptionParser(usage="usage: %prog [options] [PACKAGE]...", 
602                                description="rosmake recursively builds all dependencies before building a package", prog='rosmake') 
603          parser.add_option("--test-only", dest="test_only", default=False, 
604                            action="store_true", help="only run tests") 
605          parser.add_option("-t", dest="test", default=False, 
606                            action="store_true", help="build and test packages") 
607          parser.add_option("-a", "--all", dest="build_all", default=False, 
608                            action="store_true", help="select all packages") 
609          parser.add_option("-i", "--mark-installed", dest="mark_installed", default=False, 
610                            action="store_true", help="On successful build, mark specified packages as installed with ROS_NOBUILD") 
611          parser.add_option("-u", "--unmark-installed", dest="unmark_installed", default=False, 
612                            action="store_true", help="Remove ROS_NOBUILD from the specified packages.  This will not build anything.") 
613          parser.add_option("-v", dest="verbose", default=False, 
614                            action="store_true", help="display errored builds") 
615          parser.add_option("-r","-k", "--robust", dest="best_effort", default=False, 
616                             action="store_true", help="do not stop build on error") 
617          parser.add_option("--build-everything", dest="robust", default=False, 
618                             action="store_true", help="build all packages regardless of errors") 
619          parser.add_option("-V", dest="full_verbose", default=False, 
620                            action="store_true", help="display all builds") 
621          parser.add_option("-s", "--specified-only", dest="specified_only", default=False, 
622                            action="store_true", help="only build packages specified on the command line") 
623          parser.add_option("--buildtest", dest="buildtest", 
624                            action="append", help="package to buildtest") 
625          parser.add_option("--buildtest1", dest="buildtest1", 
626                            action="append", help="package to buildtest1") 
627          parser.add_option("--output", dest="output_dir", 
628                            action="store", help="where to output results") 
629          parser.add_option("--pre-clean", dest="pre_clean", 
630                            action="store_true", help="run make clean first") 
631          parser.add_option("--bootstrap", dest="bootstrap", default=False, 
632                            action="store_true", help="DEPRECATED, UNUSED") 
633          parser.add_option("--disable-logging", dest="logging_enabled", default=True, 
634                            action="store_false", help="turn off all logs") 
635          parser.add_option("--target", dest="target", 
636                            action="store", help="run make with this target") 
637          parser.add_option("--pjobs", dest="ros_parallel_jobs", type="int", 
638                            action="store", help="Override ROS_PARALLEL_JOBS environment variable with this number of jobs.") 
639          parser.add_option("--threads", dest="threads", type="int", default = os.environ.get("ROSMAKE_THREADS", parallel_build.num_cpus()), 
640                            action="store", help="Build up to N packages in parallel") 
641          parser.add_option("--profile", dest="print_profile", default=False, 
642                            action="store_true", help="print time profile after build") 
643          parser.add_option("--skip-blacklist", dest="skip_blacklist",  
644                            default=False, action="store_true",  
645                            help="skip packages containing a file called ROS_BUILD_BLACKLIST (Default behavior will ignore the presence of ROS_BUILD_BLACKLIST)") 
646          parser.add_option("--skip-blacklist-osx", dest="skip_blacklist_osx",  
647                            default=False, action="store_true",  
648                            help="deprecated option. it will do nothing, please use platform declarations and --require-platform instead") 
649   
650          parser.add_option("--status-rate", dest="status_update_rate", 
651                            action="store", help="How fast to update the status bar in Hz.  Default: 5Hz") 
652           
653   
654          options, args = parser.parse_args() 
655          self.printer.print_all('rosmake starting...') 
656   
657          rospack = self.rospack 
658          rosstack = self.rosstack 
659   
660          testing = False 
661          building = True 
662          if options.test_only: 
663              testing = True 
664              building = False 
665          elif options.test: 
666              testing = True 
667   
668          if options.ros_parallel_jobs: 
669              self.ros_parallel_jobs = options.ros_parallel_jobs 
670   
671          self.robust_build = options.robust 
672          self.best_effort = options.best_effort 
673          self.threads = options.threads 
674          self.skip_blacklist = options.skip_blacklist 
675          if options.skip_blacklist_osx: 
676              self.printer.print_all("Option --skip-blacklist-osx is deprecated. It will do nothing, please use platform declarations and --require-platform instead"); 
677          self.logging_enabled = options.logging_enabled 
678   
679          # pass through verbosity options 
680          self.printer.full_verbose = options.full_verbose 
681          self.printer.verbose = options.verbose 
682          if options.status_update_rate: 
683              if float(options.status_update_rate)> 0: 
684                  self.printer.duration = 1.0/float(options.status_update_rate) 
685              else: 
686                  self.printer.duration = 0 
687   
688          packages = [] 
689          #load packages from arguments 
690          if options.build_all: 
691              packages = [x for x in rospack.list() if not self.rospack.get_manifest(x).is_catkin] 
692              self.printer.print_all( "Building all packages") 
693          else:      # no need to extend if all already selected    
694              if options.buildtest: 
695                for p in options.buildtest: 
696                  packages.extend(self.rospack.get_depends_on(p))  
697                  self.printer.print_all( "buildtest requested for package %s adding it and all dependent packages: "%p) 
698   
699              if options.buildtest1: 
700                for p in options.buildtest1: 
701                  packages.extend(self.rospack.get_depends_on(p, implicit=False))  
702                  self.printer.print_all( "buildtest1 requested for package %s adding it and all depends-on1 packages: "%p) 
703   
704          if len(packages) == 0 and len(args) == 0: 
705              p = os.path.basename(os.path.abspath('.')) 
706              try: 
707                if os.path.samefile(rospack.get_path(p), '.'): 
708                  packages = [p] 
709                  self.printer.print_all( "No package specified.  Building %s"%packages) 
710                else: 
711                  self.printer.print_all("No package selected and the current directory is not the correct path for package '%s'."%p) 
712                   
713              except rospkg.ResourceNotFound as ex: 
714                  try: 
715                      stack_dir = rosstack.get_path(p) 
716                      if os.path.samefile(stack_dir, '.'): 
717                          packages = [p] 
718                          self.printer.print_all( "No package specified.  Building stack %s"%packages) 
719                      else: 
720                          self.printer.print_all("No package or stack arguments and the current directory is not the correct path for stack '%s'. Stack directory is: %s."%(p, rosstack.get_path(p))) 
721                  except: 
722                      self.printer.print_all("No package or stack specified.  And current directory '%s' is not a package name or stack name."%p) 
723          else: 
724              packages.extend(args) 
725   
726          self.printer.print_all( "Packages requested are: %s"%packages) 
727           
728          # Setup logging 
729          if self.logging_enabled: 
730            date_time_stamp =  "rosmake_output-" + time.strftime("%Y%m%d-%H%M%S") 
731            if options.output_dir: 
732                #self.log_dir = os.path.join(os.getcwd(), options.output_dir, date_time_stamp); 
733                self.log_dir = os.path.abspath(options.output_dir) 
734            else: 
735                self.log_dir = os.path.join(rospkg.get_ros_home(), "rosmake", date_time_stamp); 
736   
737            self.printer.print_all("Logging to directory %s"%self.log_dir) 
738            if os.path.exists (self.log_dir) and not os.path.isdir(self.log_dir): 
739                self.printer.print_all( "Log destination %s is a file; please remove it or choose a new destination"%self.log_dir) 
740                sys.exit(1) 
741            if not os.path.exists (self.log_dir): 
742                self.printer.print_verbose("%s doesn't exist: creating"%self.log_dir) 
743                makedirs_with_parent_perms(self.log_dir) 
744   
745            self.printer.print_verbose("Finished setting up logging") 
746   
747          stacks_arguments = [s for s in packages if s in rosstack.list()] 
748          (self.specified_packages, self.rejected_packages) = rospkg.expand_to_packages(packages, rospack, rosstack) 
749   
750          self.printer.print_all("Expanded args %s to:\n%s"%(packages, self.specified_packages)) 
751          if self.rejected_packages: 
752              self.printer.print_all("WARNING: The following args could not be parsed as stacks or packages: %s"%self.rejected_packages) 
753          if len(self.specified_packages) + len(stacks_arguments) == 0: 
754              self.printer.print_all("ERROR: No arguments could be parsed into valid package or stack names.") 
755              self.printer.running = False 
756              return False 
757   
758          if options.unmark_installed: 
759              for p in self.specified_packages: 
760                  if self.flag_tracker.remove_nobuild(p): 
761                      self.printer.print_all("Removed ROS_NOBUILD from %s"%p) 
762              self.printer.running = False 
763              return True 
764               
765          required_packages = self.specified_packages[:] 
766   
767          # catch packages of dependent stacks when specified stack is zero-sized #3528 
768          # add them to required list but not the specified list.  
769          for s in stacks_arguments: 
770              if not rosstack.packages_of(s): 
771                  for d in rosstack.get_depends(s, implicit=False): 
772                      try: 
773                          required_packages.extend(rosstack.packages_of(d)) 
774                      except ResourceNotFound: 
775                          self.printer.print_all('WARNING: The stack "%s" was not found. We will assume it is using the new buildsystem and try to continue...' % d) 
776   
777          # deduplicate required_packages 
778          required_packages = list(set(required_packages)) 
779   
780          # make sure all dependencies are satisfied and if not warn 
781          buildable_packages = [] 
782          for p in required_packages: 
783              (buildable, error, str) = self.flag_tracker.can_build(p, self.skip_blacklist, [], False) 
784              if buildable:  
785                  buildable_packages.append(p) 
786   
787          #generate the list of packages necessary to build(in order of dependencies) 
788          counter = 0 
789          for p in required_packages: 
790   
791              counter = counter + 1 
792              self.printer.print_verbose( "Processing %s and all dependencies(%d of %d requested)"%(p, counter, len(packages))) 
793              self.build_or_recurse(p) 
794   
795          # remove extra packages if specified-only flag is set 
796          if options.specified_only: 
797            new_list = [] 
798            for pkg in self.build_list: 
799              if pkg in self.specified_packages: 
800                new_list.append(pkg) 
801                self.dependency_tracker = parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack) # this will make the tracker only respond to packages in the list 
802           
803            self.printer.print_all("specified-only option was used, only building packages %s"%new_list) 
804            self.build_list = new_list 
805   
806          if options.pre_clean: 
807            build_queue = parallel_build.BuildQueue(self.build_list, parallel_build.DependencyTracker([], rospack=self.rospack), robust_build = True) 
808            self.parallel_build_pkgs(build_queue, "clean", threads = options.threads) 
809   
810          build_passed = True 
811   
812          if building: 
813            self.printer.print_verbose ("Building packages %s"% self.build_list) 
814            build_queue = parallel_build.BuildQueue(self.build_list, self.dependency_tracker, robust_build = options.robust or options.best_effort) 
815            if None not in self.result.keys(): 
816                  self.result[None] = {} 
817   
818            build_passed = self.parallel_build_pkgs(build_queue, options.target, threads = options.threads) 
819   
820          tests_passed = True 
821          if build_passed and testing: 
822              self.printer.print_verbose ("Testing packages %s"% packages) 
823              build_queue = parallel_build.BuildQueue(self.specified_packages, parallel_build.DependencyTracker(self.specified_packages, rospack=self.rospack), robust_build = True) 
824              tests_passed = self.parallel_build_pkgs(build_queue, "test", threads = 1) 
825   
826   
827          if  options.mark_installed: 
828              if build_passed and tests_passed:  
829                  for p in self.specified_packages: 
830                      if self.flag_tracker.add_nobuild(p): 
831                          self.printer.print_all("Marking %s as installed with a ROS_NOBUILD file"%p) 
832              else: 
833                  self.printer.print_all("All builds and tests did not pass cannot mark packages as installed. ") 
834   
835   
836          self.finish_time = time.time() #note: before profiling 
837          self.generate_summary_output(self.log_dir) 
838           
839          if options.print_profile: 
840              self.printer.print_all (self.get_profile_string()) 
841   
842          self.printer.running = False 
843          return build_passed and tests_passed 
844