plane_analysis.py

Go to the documentation of this file.

#!/usr/bin/env python
'''
plane_analysis.py
Mac Mason <mac@cs.duke.edu>

Perform correctness analysis on planes. This assumes that a DB is running.
'''
import sys
import roslib; roslib.load_manifest('semanticmodel')
import rospy
import hand_planes  # Where we get our ground truth from.
from semanticmodel.srv import GetCollectionNamespace
from semanticmodel.msg import Planes
from semanticmodel import point_cloud
import mongo_ros as mr
import shapely.geometry as geo

if len(sys.argv) < 2:
  print "Usage: plane_analysis <bag index> [pct = 0.75] [run_id]"
  sys.exit(1)

index = int(sys.argv[1])
pct = 0.75
min_area = 0.25 
if len(sys.argv) >= 3:
  pct = float(sys.argv[2])

run_id = None
if len(sys.argv) >= 4:
  run_id = int(sys.argv[3])

rospy.init_node('plane_analysis')

# Figure out what our database is called.
if run_id is not None:
  db_name = 'semantic_world_model'
  coll_namespace = 'run{0}'.format(run_id)
else:
  print "Waiting for get_collection_namespace service..."
  rospy.wait_for_service('get_collection_namespace')
  gcn_proxy = rospy.ServiceProxy('get_collection_namespace',
                                 GetCollectionNamespace)
  # And call.
  gcn = gcn_proxy()
  db_name = gcn.db_name
  coll_namespace = gcn.collection_namespace
  
planes_mc = mr.MessageCollection(db_name, coll_namespace+'_planes', Planes)

print "Using db {0} and collection {1}".\
      format(db_name, coll_namespace+"_planes")
planes = list(planes_mc.query({}))
print "Found {0} planes".format(len(planes))

# The data in hand_planes.meters are already in (x, y, z) meters coordinates,
# but the data in planes[0][0].planes[i].hulls are in PointCloud2 format. 

autos = []
for orig_hull in (x.hull for x in planes[0][0].planes):
  autos.append(geo.Polygon(list(point_cloud.read_points(orig_hull, 'xyz'))))

autos = [x for x in autos if x.area >= min_area]

hands = []
names = []
for k in hand_planes.meters[index]:
  hands.append(geo.Polygon(hand_planes.meters[index][k]))
  names.append(k)

#for (hand, name) in zip(hands, names):
#  print "%s has area %f" % (name, hand.area)

pairs = [(x, y) for (x, y) in zip(hands, names) if x.area >= min_area]
hands = [x[0] for x in pairs]
names = [x[1] for x in pairs]

print "After area culling:", len(autos), "autos and", len(hands), "hands."

# See Section 7 of the paper for definitions.
def approx_subset(a, b):
  '''
  Is a an approximate subset of b, according to section 7 of the paper?
  '''
  ai = a.intersection(b).area
  return ai >= pct * a.area

def approx_superset(a, b):
  '''
  I am an approximate superset of you if you're an approximate subset of me.
  '''
  return approx_subset(b, a)

def approx_equals(a, b):
  return approx_subset(a, b) and approx_superset(a, b)

def intersects(a, b):
  '''Does a intersect b at all?'''
  return a.intersection(b).area > 0.0

def categorize(autos, hands, index):
  '''
  Given the set of autos, and hands, and an index (into autos), return the
  appropriate categorization of that index. This always returns a tuple whose
  first element is in range(1, 6), describing the category; the next value is
  the thing it matched against (but see the last return).
  '''
  for i, hand in enumerate(hands):
    if approx_equals(autos[index], hand):
      return (1, i)  # autos[index] matched hands[i].
  # Not approximately equal to anything.
  for i, hand in enumerate(hands):
    if approx_subset(autos[index], hand):
      return (2, i, index)
  # Not equal or an approx_subset.
  for i, hand in enumerate(hands):
    if approx_superset(autos[index], hand):
      return (3, i, index)
  # Not equal, sub, or super.
  for i, hand in enumerate(hands):
    if intersects(autos[index], hand):
      return (4, i, index)
  # No intersection at all!
  return (5, index)  # Second value just makes printing easier

def false_negatives(autos, hands):
  '''
  Find the hands that have no autos intersecting them at all.
  '''
  res = []
  for i, hand in enumerate(hands):
    hit = False
    for auto in autos:
      if hand.intersection(auto).area > 0.0:
        hit = True
    if not hit:
      res.append(i)
  return res

categorized = [categorize(autos, hands, i) for i in range(len(autos))]

truepos = [x for x in categorized if x[0] == 1]
print len(truepos), "True positives:"
for i, tp in enumerate(truepos):
  print "    %02d: %s" % (i+1, names[tp[1]])

print ""
oversegs = [x for x in categorized if x[0] == 2]
print len(oversegs), "Oversegmentations: (intersection / auto, / hand)"
for i, os in enumerate(oversegs):
  aa = autos[os[2]].area
  ah = hands[os[1]].area
  ai = autos[os[2]].intersection(hands[os[1]]).area
  print "    %02d: %s: %f %f" % (i+1, names[os[1]], ai / aa, ai / ah)

print ""
undersegs = [x for x in categorized if x[0] == 3]
print len(undersegs), "Undersegmentations: (intersection / auto, / hand)"
for i, us in enumerate(undersegs):
  aa = autos[us[2]].area
  ah = hands[us[1]].area
  ai = autos[us[2]].intersection(hands[us[1]]).area
  print "    %02d: %s: %f %f" % (i+1, names[us[1]], ai / aa, ai / ah)

print ""
intersections = [x for x in categorized if x[0] == 4]
print len(intersections), "Intersections: (intersection / auto, / hand)"
for i, inter in enumerate(intersections):
  aa = autos[inter[2]].area
  ah = hands[inter[1]].area
  ai = autos[inter[2]].intersection(hands[inter[1]]).area
  print "    %02d: %s: %f %f" % (i+1, names[inter[1]], ai / aa, ai / ah)

print ""
fps = [x for x in categorized if x[0] == 5]
print len(fps), "False positives: (area)"
for i, fp in enumerate(fps):
  print "    %02d: %f" % (i+1, autos[fp[1]].area)

print ""
fns = false_negatives(autos, hands)
print len(fns), "False negatives:"
for i, fn in enumerate(fns):
  print "    %02d: %s: %f" % (i+1, names[fn], hands[fn].area)

# Now, let's figure out (a) our total areas and (b) the total area covered. 
handtotal = sum([x.area for x in hands])
autototal = sum([x.area for x in autos])
print ""
print "Total hand area: %f Total auto area: %f" % (handtotal, autototal)
it = 0.0
for hand in hands:
  for auto in autos:
    it += hand.intersection(auto).area
print "Intersection total: %f (%f%% of gt)" % (it, 100.0 * it / handtotal)

# Now, how much area did we find that isn't part of gt?
mistake = 0.0
for i, auto in enumerate(autos):
  a = auto.area
  for j, hand in enumerate(hands):
    a -= auto.intersection(hand).area
  mistake += max([a, 0.0])

print "Error area:", mistake, "percent of total:", 100.0 * mistake / autototal