pycv.cs.cv.detect

1 # PyCV - A Computer Vision Package for Python Incorporating Fast Training of Face Detection 2 3 # Copyright 2007 Nanyang Technological University, Singapore. 4 # Authors: Minh-Tri Pham, Viet-Dung D. Hoang, and Tat-Jen Cham. 5 6 # This file is part of PyCV. 7 8 # PyCV is free software: you can redistribute it and/or modify 9 # it under the terms of the GNU General Public 10 # License as published by the Free Software Foundation, either version 11 # 3 of the License, or (at your option) any later version. 12 13 # PyCV is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 18 # You should have received a copy of the GNU General Public License 19 # along with this program. If not, see <http://www.gnu.org/licenses/>. 20 21 # --------------------------------------------------------------------- 22 #!/usr/bin/env python 23 24 25 __all__ = ['ObjectDetector', 'ObjectDetectionInfo'] 26 27 28 from cPickle import load 29 from ctypes import pointer, byref 30 from numpy import array, zeros, concatenate, ones 31 from scipy.weave import inline 32 from time import time 33 from cv import fromIplImage 34 from pycv.cs.ml.cla.boost import DiscreteBoostedClassifier, SimpleCascade, \ 35 GeneralizedCascade 36 #from pycv import tprint 37 from pycv.interfaces.opencv import cvLoadCast, CvHaarClassifierCascade, \ 38 cvCreateMemStorage, cvReleaseMemStorage, cvReleaseHaarClassifierCascade, \ 39 cvHaarDetectObjects, CvSize, cvGetSeqElem, CvRect, haar_cascades, \ 40 cvReleaseImage, cvCopy, cvResize, CV_INTER_NN, IPL_DEPTH_8U, \ 41 IPL_DEPTH_32S, IPL_DEPTH_64F, cvIntegral, cvSet, cvSetZero, CvScalar, \ 42 cvCreateImage, cvCvtColor, cvSetImagesForHaarClassifierCascade, \ 43 cvRunHaarClassifierCascade, cvSetImageROI, cvResetImageROI, CvPoint, \ 44 cvResizeSubRect, cvGetSize, CvRECT, cvRectangle, CV_CVTIMG_FLIP, \ 45 CV_RGB2GRAY, CV_BGR2GRAY, cvFlip, CvMat, cvGetMat 46 47 from haar import THaarClassifier 48 49 from integralimage import integral, integral_uint32, integral_int32 50 51 from pycv.ext import detect_detect_block, detect_compute_ivecs2 52 53

54 -class ObjectDetectionInfo:

55 ivecs = None 56 dvecs = None 57 patch_len = None 58

59 - def save_to_file(self,filepath):

60 f = open(filepath,'w') 61 f.write('========DATA========\n') 62 M = len(self.ivecs) 63 f.write(str(M)+' '+str(self.patch_len)+'\n') 64 for m in xrange(M): 65 for i in xrange(64): 66 f.write(str(self.ivecs[m,i])+' ') 67 for i in xrange(4): 68 f.write(str(self.dvecs[m,i])+' ') 69 f.write('\n')

70

71 - def load_from_file(self,filepath):

72 f = open(filepath,'r') 73 s = f.read().split() 74 while s.pop(0) != '========DATA========': 75 pass 76 M = int(s.pop(0)) # len(self.ivecs) 77 patch_len = int(s.pop(0)) 78 ivecs = zeros((M,64),'int') 79 dvecs = zeros((M,4),'double') 80 for m in xrange(M): 81 for i in xrange(64): 82 ivecs[m,i] = int(s.pop(0)) 83 for i in xrange(4): 84 dvecs[m,i] = float(s.pop(0)) 85 86 self.ivecs = ivecs 87 self.dvecs = dvecs 88 self.patch_len = patch_len

89

90 - def load_from_classifier(classifier, patch_len):

91 self.ivecs, self.dvecs = self._convert_Haars(classifier, patch_len) 92 self.patch_len = patch_len

93

94 - def copy_from(self, ivecs, dvecs, patch_len):

95 self.ivecs = ivecs 96 self.dvecs = dvecs 97 self.patch_len = patch_len

98

99 - def _convert_Haar(self,lc,patch_len,c,is_improver,to_reset,reset_value=0):

100 """Convert from THaarClassifier to a new classifier with scale s so 101 that I can detect from image with different scales. 102 103 Input: 104 lc: a THaarClassifier 105 patch_len: the length of the patch to be used by lc 106 c: the voting weight (double) 107 is_improver: (boolean) an improver or a filter? 108 to_reset: (boolean) to reset previous sum? 109 reset_value: (double) value to be reset to 110 Output: 111 all the parameters of the haar classifier will be stored in an int 112 array and a double array 113 114 New reference: 115 116 ivec: 64 items, format as follows 117 - ivec[0]: the number of indices 118 - ivec[1]: flag, in which: 119 - bit 0: set if it is an improver, otherwise it is a filter 120 - bit 1: set if the previous sum is to be reset 121 - starting from ivec[4]: tuples of 3 integers: 122 - x position w.r.t. the top-left corner 123 - y position w.r.t. the top-left corner 124 - coefficient/weight 125 126 dvec: 4 items, format as follows 127 - dvec[0]: threshold b 128 - dvec[1]: voting weight c 129 - dvec[2]: reset value -- only meaningful if bit 1 of ivec[1] is set 130 131 Old reference: 132 133 ivec: format as follows 134 - first 32 ints: (vectorized) indices w.r.t. the location 135 of the top-left corner 136 - ivec[32]: the number of indices 137 - ivec[33]: flag, in which: 138 - bit 0: set if it is an improver, otherwise it is a filter 139 - bit 1: set if the previous sum is to be reset 140 141 dvec: format as follows 142 - first 32 doubles: weights 143 - dvec[32]: threshold b 144 - dvec[33]: voting weight c 145 - dvec[34]: reset value -- only meaningful if bit 1 of ivec[33] is set 146 """ 147 ivec = zeros(64,'int') 148 dvec = zeros(4,'double') 149 150 # for debugging 151 ## ivec[2] = patch_len 152 153 N = len(lc.p) 154 ivec[0] = N 155 for n in xrange(N): 156 ivec[n*3+4] = lc.p[n] % patch_len 157 ivec[n*3+5] = lc.p[n] / patch_len 158 ivec[n*3+6] = int(round(lc.w[n])) 159 160 dvec[0] = lc.b 161 dvec[1] = c 162 dvec[2] = reset_value 163 164 if is_improver: ivec[1] |= 1 # set bit 0 165 if to_reset: ivec[1] |= 2 # set bit 1 166 167 return ivec, dvec

168

169 - def _convert_Haars(self,cc,patch_len):

170 """Convert from composite to a new classifier with scale s so that I can detect from image with different scales. 171 172 Input: 173 cc: a composite classifier, which can be: 174 - A THaarClassifier 175 - A SimpleCascade of composite classifiers 176 - A GeneralizedCascade of composite classifiers 177 - A DiscreteBoostedClassifier of composite classifiers 178 patch_len: the length of the patch to be used by lc 179 Output: 180 ivecs: a 2D int array representing the 'int' parameters of the composite classifier 181 dvecs: a 2D double array representing the 'double' parameters of the composite classifier 182 """ 183 if isinstance(cc,THaarClassifier): 184 ivec, dvec = self._convert_Haar(cc,patch_len,1,False,True) 185 return ivec.reshape(1,64), dvec.reshape(1,4) 186 187 if isinstance(cc,DiscreteBoostedClassifier): 188 for weak in cc.weaks: 189 if not isinstance(weak,THaarClassifier): 190 raise ValueError("cc is a DiscreteBoostedClassifier, but one of its weak classifiers is not THaarClassifier") 191 N = len(cc.weaks) 192 ivecs = zeros((N,64),'int') 193 dvecs = zeros((N,4),'double') 194 for n in xrange(N): 195 ivecs[n], dvecs[n] = self._convert_Haar(cc.weaks[n],patch_len,cc.c[n],n != N-1,n == 0,0) 196 return ivecs, dvecs 197 198 if isinstance(cc,SimpleCascade): 199 for weak in cc.bclassifiers: 200 if not isinstance(weak,THaarClassifier) and \ 201 not isinstance(weak,DiscreteBoostedClassifier) and \ 202 not isinstance(weak,GeneralizedCascade): 203 raise ValueError("cc is a SimpleCascade, but one of its weak \ 204 classifiers is neither a THaarClassifier, nor a \ 205 DiscreteBoostedClassifier, nor a GeneralizedCascade") 206 207 z = [self._convert_Haars(x,patch_len) for x in cc.bclassifiers] 208 ivecs = concatenate([x[0] for x in z]) 209 dvecs = concatenate([x[1] for x in z]) 210 return ivecs, dvecs 211 212 if isinstance(cc,GeneralizedCascade): 213 N = len(cc.weaks) 214 for n in xrange(1,N): 215 if not isinstance(cc.weaks[n],THaarClassifier): 216 raise ValueError("cc is a GeneralizedCascade, but one of its weak classifiers is not a THaarClassifier") 217 ivecs = zeros((N-1,64),'int') 218 dvecs = zeros((N-1,4),'double') 219 ivecs[0], dvecs[0] = self._convert_Haar(cc.weaks[1],patch_len,cc.c[1],cc.z[1] and 1 != N,True,1) 220 for n in xrange(2,N): 221 ivecs[n-1], dvecs[n-1] = self._convert_Haar(cc.weaks[n],patch_len,cc.c[n],cc.z[n] and n != N-1,False,0) 222 return ivecs, dvecs 223 224 raise ValueError("Unknown class type of cc")

225 226

227 -class ObjectDetector:

228

229 - def __init__(self, object_type, flag = 0, param1 = None, param2 = None, 230 block_size=256):

231 """Initialize an ObjectDetector. 232 233 Input: 234 object_type: the type of object to detect, 235 in ('frontal_face','profile_face','upper_body','lower_body','full_body') 236 flag: the type of operation of the object detector 237 0 = use opencv's default haar cascade 238 1 = use the cascade stored in a BinaryClassifier, see `param1`, `param2` 239 2 = same as 1 but use opencv's default haar cascade to post-verify 240 3 = use the cascade stored in a text file named by `param1` 241 4 = same as 3 but use opencv's default haar cascade to post-verify 242 param1: [optional] a strong classifier to classify a patch into object/non-object 243 If flag is 1 or 2: This holds a BinaryClassifier 244 If flag is 3 or 4: This holds a path to the text file storing the cascade 245 param2: [optional] If flag is 1 or 2: 246 this holds the length of the image patch (e.g. 20-by-20 or 24-by-24) 247 block_size : int 248 length of a square block. An image is divided into 249 blocks if it is too large. This offers fast and stable 250 object detection. 251 252 """ 253 self.has_classifier = flag != 0 254 self.use_opencv = flag in (0,2,4) 255 self.block_size = block_size 256 257 self.N = None 258 259 if flag in (1,2): # load from the classifier 260 self.odi = ObjectDetectionInfo() 261 self.odi.load_from_classifier(param1, param2) 262 self.ivecs = self.odi.ivecs 263 self.dvecs = self.odi.dvecs 264 self.patch_len = self.odi.patch_len 265 elif flag in (3,4): # load from file 266 self.odi = ObjectDetectionInfo() 267 self.odi.load_from_file(param1) 268 self.ivecs = self.odi.ivecs 269 self.dvecs = self.odi.dvecs 270 self.patch_len = self.odi.patch_len 271 272 if self.has_classifier: 273 size = CvSize(block_size, block_size) 274 self.img = cvCreateImage( size, IPL_DEPTH_8U, 1 ) 275 self.pimg = fromIplImage(self.img) 276 self.mask = zeros((block_size,block_size), 'uint8') 277 self.img_cum = zeros((block_size,block_size),'int32') 278 self.org_img = None 279 280 # compute ivecs2 281 self.ivecs2 = zeros(self.ivecs.shape, 'int') 282 img_ws = int(self.img_cum.strides[0] / 4) # sizeof(int32) 283 detect_compute_ivecs2(img_ws, self.ivecs, self.ivecs2) 284 285 self.q = zeros((2,10000),'int') 286 self.out = zeros((2,1000),'int') 287 288 289 290 if self.use_opencv: 291 self.haar_cascade = cvLoadCast(haar_cascades[object_type],CvHaarClassifierCascade) 292 self.storage = cvCreateMemStorage(0) 293 294 if self.has_classifier: 295 size = CvSize(block_size+1, block_size+1) 296 self.img_sum = cvCreateImage( size, IPL_DEPTH_32S, 1 ) 297 self.img_sqsum = cvCreateImage( size, IPL_DEPTH_64F, 1 ) 298 self.img_tlsum = cvCreateImage( size, IPL_DEPTH_32S, 1 )

299

300 - def __del__(self):

301 if self.use_opencv: 302 if self.has_classifier: 303 cvReleaseImage(self.img_tlsum) 304 cvReleaseImage(self.img_sqsum) 305 cvReleaseImage(self.img_sum) 306 307 cvReleaseMemStorage(byref(self.storage)) 308 cvReleaseHaarClassifierCascade(byref(self.haar_cascade)) 309 310 if self.has_classifier: 311 cvReleaseImage(self.img) 312 313 if self.org_img is not None: 314 cvReleaseImage(self.org_img)

315 316

317 - def _detect_block(self, roi):

318 """Detect objects in the current block. 319 320 :Parameters: 321 roi : CvRECT 322 rectangle of interest of the top-left corner 323 324 :Returns: 325 array of the top-left corner(s) of the detected face(s) 326 """ 327 # initialize variables 328 mask = self.mask 329 img = self.img_cum 330 rect = array((roi.x,roi.y,roi.x+roi.w,roi.y+roi.h)).astype('int') 331 332 min_neighbors = int(self.min_neighbors) 333 plen = int(self.patch_len) 334 335 ivecs = self.ivecs 336 ivecs2 = self.ivecs2 337 dvecs = self.dvecs 338 339 qx = self.q[0] 340 qy = self.q[1] 341 outx = self.out[0] 342 outy = self.out[1] 343 344 # find bound 345 w = rect[2]+plen 346 h = rect[3]+plen 347 348 # integrate 349 integral_int32(self.pimg[:h,:w], self.img_cum[:h,:w]) 350 351 # detect 352 nout = detect_detect_block(mask,img,ivecs2,dvecs, \ 353 min_neighbors,qx,qy,outx,outy,rect) 354 355 z = [] 356 357 # finalize the candidates 358 if nout > 0 and self.use_opencv: 359 cvIntegral( self.img, self.img_sum, self.img_sqsum, self.img_tlsum ) 360 cvSetImagesForHaarClassifierCascade(self.haar_cascade, \ 361 self.img_sum,self.img_sqsum,self.img_tlsum,1.0) 362 363 for i in xrange(nout): 364 x1 = int(outx[i]) 365 y1 = int(outy[i]) 366 367 if self.use_opencv and \ 368 cvRunHaarClassifierCascade(self.haar_cascade, CvPoint(x1,y1)) <= 0: 369 continue 370 371 z.append((x1,y1,plen,plen)) 372 373 return z

374

375 - def _display_block(self, roi):

376 """Diplay the current block. 377 378 :Parameters: 379 roi : CvRECT 380 rectangle of interest of the top-left corner 381 382 :Returns: 383 array of the top-left corner(s) of the detected face(s) 384 """ 385 cvRectangle(self.img, CvPoint(int(roi.x), int(roi.y)), 386 CvPoint(int(roi.x+roi.w), int(roi.y+roi.h)), CvScalar(255,0,0,0),3) 387 cvShowImage('screen2', self.img) 388 cvWaitKey(0) 389 return []

390

391 - def _update_org_img(self, img):

392 393 if self.org_img is None: 394 self.org_img = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 1) 395 elif self.org_img.contents.width != img.contents.width or \ 396 self.org_img.contents.height != img.contents.height: 397 cvReleaseImage(self.org_img) 398 self.org_img = cvCreateImage(cvGetSize(img), IPL_DEPTH_8U, 1) 399 400 if img[0].origin > 0: 401 mat = byref(CvMat()) 402 cvGetMat(self.org_img, mat, None, 0) 403 if img[0].nChannels > 1: 404 cvCvtColor(img, mat, CV_BGR2GRAY) 405 else: 406 cvCopy(img, mat) 407 cvFlip(mat) 408 else: 409 if img[0].nChannels > 1: 410 cvCvtColor(img, self.org_img, CV_BGR2GRAY) 411 else: 412 cvCopy(img, self.org_img)

413

414 - def _group_overlapping(self, z):

415 """Group overlapping windows 416 417 :Paramters: 418 z : array of (x,y,w,h) 419 array of windows 420 421 :Returns: 422 y : array of (x,y,w,h) 423 array of windows after groupping 424 """ 425 426 def overlapping(i,j): 427 x1 = max(z[i,0],z[j,0]) 428 y1 = max(z[i,1],z[j,1]) 429 x2 = min(z[i,0]+z[i,2],z[j,0]+z[j,2]) 430 y2 = min(z[i,1]+z[i,3],z[j,1]+z[j,3]) 431 if x1 >= x2 or y1 >= y2: 432 return False 433 a = (x2-x1)*(y2-y1) 434 return a >= 0.5*min(z[i,2]*z[i,3], z[j,2]*z[j,3])

435 436 mask = ones(len(z)) 437 z2 = [] 438 for i in xrange(len(z)): 439 if mask[i]: 440 x = z[i,0] 441 y = z[i,1] 442 w = z[i,2] 443 h = z[i,3] 444 n = 1 445 mask[i] = 0 446 for j in xrange(i+1,len(z)): 447 if mask[j]==1 and overlapping(i,j): 448 x += z[j,0] 449 y += z[j,1] 450 w += z[j,2] 451 h += z[j,3] 452 n += 1 453 mask[j] = 0 454 455 z2.append(array((x,y,w,h))/n) 456 457 return array(z2)

458

459 - def detect(self, img, scale_factor=1.1, min_neighbors=1, 460 group_overlapping=True):

461 """Detect objects from a grayscale image. 462 463 Input: 464 img: a POINTER(IplImage) of depth IPL_DEPTH_8U and 1 or 3 channels 465 scale_factor: The factor by which the search window is scaled 466 between the subsequent scans, for example, 1.1 means increasing window by 10%. 467 min_neighbors: Minimum number (minus 1) of neighbor rectangles that makes up an object. 468 All the groups of a smaller number of rectangles than min_neighbors-1 are rejected. 469 If min_neighbors is 0, the function does not any grouping at all and returns all 470 the detected candidate rectangles, which may be useful if the user wants to apply 471 a customized grouping procedure. 472 group_overlapping: if True then overlapping locations are grouped 473 Output: 474 z: an 'int' numpy.array of tuples (x,y,w,h) representing the object locations 475 """ 476 if self.has_classifier: # got a classifier 477 self.min_neighbors = min_neighbors 478 self._update_org_img(img) 479 480 self.faces = [] 481 482 # src = view on self.org_img, dst = block 483 src_w = self.org_img.contents.width 484 src_h = self.org_img.contents.height 485 dst_size = self.block_size-self.patch_len 486 487 scale = 1.0 488 489 while True: 490 src_size = dst_size*scale 491 src_outer_size = self.block_size*scale 492 493 for iy in xrange(int(src_h/src_size)+1): 494 src_y = iy*src_size 495 dst_h = min((src_h-src_y)/scale, dst_size) 496 for ix in xrange(int(src_w/src_size)+1): 497 src_x = ix*src_size 498 dst_w = min((src_w-src_x)/scale, dst_size) 499 500 rect = CvRECT(src_x, src_y, 501 src_outer_size, src_outer_size) 502 cvResizeSubRect(self.org_img, self.img, rect) 503 504 roi = CvRECT(0, 0, dst_w, dst_h) 505 z = self._detect_block(roi) 506 if len(z) > 0: 507 z = array(z)*scale 508 z[:,0] += src_x 509 z[:,1] += src_y 510 self.faces.extend(z) 511 512 ## # Try this for a display of blocks 513 ## self._display_block(roi) 514 515 if min(src_w,src_h) / scale < self.patch_len*2: 516 break 517 scale *= scale_factor 518 519 z = array(self.faces) 520 521 else: # use opencv's default slow haar_cascade 522 faces = cvHaarDetectObjects( img, self.haar_cascade, self.storage, scale_factor, 523 min_neighbors, 0, CvSize(0,0) ) 524 525 z = zeros((len(faces),4),'int') 526 for i in xrange(len(faces)): 527 z[i,0] = faces[i].x 528 z[i,1] = faces[i].y 529 z[i,2] = faces[i].width 530 z[i,3] = faces[i].height 531 532 if group_overlapping: 533 z = self._group_overlapping(z) 534 535 return z.astype('int')

536

Source Code for Module pycv.cs.cv.detect