import pickle from os import path as osp import mmcv import numpy as np from mmcv import track_iter_progress from mmcv.ops import roi_align from pycocotools import mask as maskUtils from pycocotools.coco import COCO from mmdet3d.core.bbox import box_np_ops as box_np_ops from mmdet3d.datasets import build_dataset from mmdet.core.evaluation.bbox_overlaps import bbox_overlaps def _poly2mask(mask_ann, img_h, img_w): if isinstance(mask_ann, list): # polygon -- a single object might consist of multiple parts # we merge all parts into one mask rle code rles = maskUtils.frPyObjects(mask_ann, img_h, img_w) rle = maskUtils.merge(rles) elif isinstance(mask_ann["counts"], list): # uncompressed RLE rle = maskUtils.frPyObjects(mask_ann, img_h, img_w) else: # rle rle = mask_ann mask = maskUtils.decode(rle) return mask def _parse_coco_ann_info(ann_info): gt_bboxes = [] gt_labels = [] gt_bboxes_ignore = [] gt_masks_ann = [] for i, ann in enumerate(ann_info): if ann.get("ignore", False): continue x1, y1, w, h = ann["bbox"] if ann["area"] <= 0: continue bbox = [x1, y1, x1 + w, y1 + h] if ann.get("iscrowd", False): gt_bboxes_ignore.append(bbox) else: gt_bboxes.append(bbox) gt_masks_ann.append(ann["segmentation"]) if gt_bboxes: gt_bboxes = np.array(gt_bboxes, dtype=np.float32) gt_labels = np.array(gt_labels, dtype=np.int64) else: gt_bboxes = np.zeros((0, 4), dtype=np.float32) gt_labels = np.array([], dtype=np.int64) if gt_bboxes_ignore: gt_bboxes_ignore = np.array(gt_bboxes_ignore, dtype=np.float32) else: gt_bboxes_ignore = np.zeros((0, 4), dtype=np.float32) ann = dict(bboxes=gt_bboxes, bboxes_ignore=gt_bboxes_ignore, masks=gt_masks_ann) return ann def crop_image_patch_v2(pos_proposals, pos_assigned_gt_inds, gt_masks): import torch from torch.nn.modules.utils import _pair device = pos_proposals.device num_pos = pos_proposals.size(0) fake_inds = torch.arange(num_pos, device=device).to(dtype=pos_proposals.dtype)[ :, None ] rois = torch.cat([fake_inds, pos_proposals], dim=1) # Nx5 mask_size = _pair(28) rois = rois.to(device=device) gt_masks_th = ( torch.from_numpy(gt_masks) .to(device) .index_select(0, pos_assigned_gt_inds) .to(dtype=rois.dtype) ) # Use RoIAlign could apparently accelerate the training (~0.1s/iter) targets = roi_align(gt_masks_th, rois, mask_size[::-1], 1.0, 0, True).squeeze(1) return targets def crop_image_patch(pos_proposals, gt_masks, pos_assigned_gt_inds, org_img): num_pos = pos_proposals.shape[0] masks = [] img_patches = [] for i in range(num_pos): gt_mask = gt_masks[pos_assigned_gt_inds[i]] bbox = pos_proposals[i, :].astype(np.int32) x1, y1, x2, y2 = bbox w = np.maximum(x2 - x1 + 1, 1) h = np.maximum(y2 - y1 + 1, 1) mask_patch = gt_mask[y1 : y1 + h, x1 : x1 + w] masked_img = gt_mask[..., None] * org_img img_patch = masked_img[y1 : y1 + h, x1 : x1 + w] img_patches.append(img_patch) masks.append(mask_patch) return img_patches, masks def create_groundtruth_database( dataset_class_name, data_path, info_prefix, info_path=None, mask_anno_path=None, used_classes=None, database_save_path=None, db_info_save_path=None, relative_path=True, add_rgb=False, lidar_only=False, bev_only=False, coors_range=None, with_mask=False, load_augmented=None, ): """Given the raw data, generate the ground truth database. Args: dataset_class_name （str): Name of the input dataset. data_path (str): Path of the data. info_prefix (str): Prefix of the info file. info_path (str): Path of the info file. Default: None. mask_anno_path (str): Path of the mask_anno. Default: None. used_classes (list[str]): Classes have been used. Default: None. database_save_path (str): Path to save database. Default: None. db_info_save_path (str): Path to save db_info. Default: None. relative_path (bool): Whether to use relative path. Default: True. with_mask (bool): Whether to use mask. Default: False. """ print(f"Create GT Database of {dataset_class_name}") dataset_cfg = dict( type=dataset_class_name, dataset_root=data_path, ann_file=info_path ) if dataset_class_name == "KittiDataset": dataset_cfg.update( test_mode=False, split="training", modality=dict( use_lidar=True, use_depth=False, use_lidar_intensity=True, use_camera=with_mask, ), pipeline=[ dict( type="LoadPointsFromFile", coord_type="LIDAR", load_dim=4, use_dim=4, ), dict( type="LoadAnnotations3D", with_bbox_3d=True, with_label_3d=True, ), ], ) elif dataset_class_name == "NuScenesDataset": if not load_augmented: dataset_cfg.update( use_valid_flag=True, pipeline=[ dict( type="LoadPointsFromFile", coord_type="LIDAR", load_dim=5, use_dim=5, ), dict( type="LoadPointsFromMultiSweeps", sweeps_num=10, use_dim=[0, 1, 2, 3, 4], pad_empty_sweeps=True, remove_close=True, ), dict( type="LoadAnnotations3D", with_bbox_3d=True, with_label_3d=True ), ], ) else: dataset_cfg.update( use_valid_flag=True, pipeline=[ dict( type="LoadPointsFromFile", coord_type="LIDAR", load_dim=16, use_dim=list(range(16)), load_augmented=load_augmented, ), dict( type="LoadPointsFromMultiSweeps", sweeps_num=10, load_dim=16, use_dim=list(range(16)), pad_empty_sweeps=True, remove_close=True, load_augmented=load_augmented, ), dict( type="LoadAnnotations3D", with_bbox_3d=True, with_label_3d=True ), ], ) elif dataset_class_name == "WaymoDataset": dataset_cfg.update( test_mode=False, split="training", modality=dict( use_lidar=True, use_depth=False, use_lidar_intensity=True, use_camera=False, ), pipeline=[ dict( type="LoadPointsFromFile", coord_type="LIDAR", load_dim=6, use_dim=5, ), dict( type="LoadAnnotations3D", with_bbox_3d=True, with_label_3d=True, ), ], ) dataset = build_dataset(dataset_cfg) if database_save_path is None: database_save_path = osp.join(data_path, f"{info_prefix}_gt_database") if db_info_save_path is None: db_info_save_path = osp.join(data_path, f"{info_prefix}_dbinfos_train.pkl") mmcv.mkdir_or_exist(database_save_path) all_db_infos = dict() if with_mask: coco = COCO(osp.join(data_path, mask_anno_path)) imgIds = coco.getImgIds() file2id = dict() for i in imgIds: info = coco.loadImgs([i])[0] file2id.update({info["file_name"]: i}) group_counter = 0 for j in track_iter_progress(list(range(len(dataset)))): input_dict = dataset.get_data_info(j) dataset.pre_pipeline(input_dict) example = dataset.pipeline(input_dict) annos = example["ann_info"] image_idx = example["sample_idx"] points = example["points"].tensor.numpy() gt_boxes_3d = annos["gt_bboxes_3d"].tensor.numpy() names = annos["gt_names"] group_dict = dict() if "group_ids" in annos: group_ids = annos["group_ids"] else: group_ids = np.arange(gt_boxes_3d.shape[0], dtype=np.int64) difficulty = np.zeros(gt_boxes_3d.shape[0], dtype=np.int32) if "difficulty" in annos: difficulty = annos["difficulty"] num_obj = gt_boxes_3d.shape[0] point_indices = box_np_ops.points_in_rbbox(points, gt_boxes_3d) if with_mask: # prepare masks gt_boxes = annos["gt_bboxes"] img_path = osp.split(example["img_info"]["filename"])[-1] if img_path not in file2id.keys(): print(f"skip image {img_path} for empty mask") continue img_id = file2id[img_path] kins_annIds = coco.getAnnIds(imgIds=img_id) kins_raw_info = coco.loadAnns(kins_annIds) kins_ann_info = _parse_coco_ann_info(kins_raw_info) h, w = annos["img_shape"][:2] gt_masks = [_poly2mask(mask, h, w) for mask in kins_ann_info["masks"]] # get mask inds based on iou mapping bbox_iou = bbox_overlaps(kins_ann_info["bboxes"], gt_boxes) mask_inds = bbox_iou.argmax(axis=0) valid_inds = bbox_iou.max(axis=0) > 0.5 # mask the image # use more precise crop when it is ready # object_img_patches = np.ascontiguousarray( # np.stack(object_img_patches, axis=0).transpose(0, 3, 1, 2)) # crop image patches using roi_align # object_img_patches = crop_image_patch_v2( # torch.Tensor(gt_boxes), # torch.Tensor(mask_inds).long(), object_img_patches) object_img_patches, object_masks = crop_image_patch( gt_boxes, gt_masks, mask_inds, annos["img"] ) for i in range(num_obj): filename = f"{image_idx}_{names[i]}_{i}.bin" abs_filepath = osp.join(database_save_path, filename) rel_filepath = osp.join(f"{info_prefix}_gt_database", filename) # save point clouds and image patches for each object gt_points = points[point_indices[:, i]] gt_points[:, :3] -= gt_boxes_3d[i, :3] if with_mask: if object_masks[i].sum() == 0 or not valid_inds[i]: # Skip object for empty or invalid mask continue img_patch_path = abs_filepath + ".png" mask_patch_path = abs_filepath + ".mask.png" mmcv.imwrite(object_img_patches[i], img_patch_path) mmcv.imwrite(object_masks[i], mask_patch_path) with open(abs_filepath, "w") as f: gt_points.tofile(f) if (used_classes is None) or names[i] in used_classes: db_info = { "name": names[i], "path": rel_filepath, "image_idx": image_idx, "gt_idx": i, "box3d_lidar": gt_boxes_3d[i], "num_points_in_gt": gt_points.shape[0], "difficulty": difficulty[i], } local_group_id = group_ids[i] # if local_group_id >= 0: if local_group_id not in group_dict: group_dict[local_group_id] = group_counter group_counter += 1 db_info["group_id"] = group_dict[local_group_id] if "score" in annos: db_info["score"] = annos["score"][i] if with_mask: db_info.update({"box2d_camera": gt_boxes[i]}) if names[i] in all_db_infos: all_db_infos[names[i]].append(db_info) else: all_db_infos[names[i]] = [db_info] for k, v in all_db_infos.items(): print(f"load {len(v)} {k} database infos") with open(db_info_save_path, "wb") as f: pickle.dump(all_db_infos, f)