Update

Author: vghost2008

dataset_tools/transcrowdpose2coco.py

@@ -27,7 +27,7 @@
     for path in file_path:
         data_t = read_crowd_pose(path)
         datas.extend(data_t)
-    do_vis = True
+    do_vis = False
     new_coco_data = []
     for i,data in enumerate(datas):
         sys.stdout.write(f"\r{i}/{len(datas)}")
@@ -52,7 +52,7 @@
             wmli.imwrite(save_path,img)
         new_coco_data.append([img_name,org_bboxes,kps])
 
-    coco_pt_path = '/home/wj/ai/mldata1/crowd_pose/CrowdPose/crowdpose_coco.pt',
+    coco_pt_path = '/home/wj/ai/mldata1/crowd_pose/CrowdPose/crowdpose_coco.pt'
     with open(coco_pt_path,"wb") as f:
         pickle.dump(new_coco_data,f)
     exit(0)

dataset_tools/transpennaction2cocov2.py

@@ -30,18 +30,19 @@
         else:
             datas = datas[eidx:bidx]
             datas = datas[::-1]
-    do_vis = False
+    do_vis = True
     if not os.path.exists(coco_pt_dir):
         os.makedirs(coco_pt_dir)
 
     for i,data in enumerate(datas):
         sys.stdout.write(f"\r{i}/{len(datas)}")
         file,kps,bboxes = data
         bf_name = wmlu.base_name(file)
-        coco_pt_path = f"/home/wj/ai/mldata1/penn_action/Penn_Action/coco_labels/{bf_name}.pt"
+        '''coco_pt_path = f"/home/wj/ai/mldata1/penn_action/Penn_Action/coco_labels/{bf_name}.pt"
         if osp.exists(coco_pt_path):
-            continue
+            continue'''
         new_coco_data = []
+        left_node = list(range(1,17,2))
         for i,kp,bbox in zip(count(),kps,bboxes):
             img_name = f"{bf_name}/{i+1:06d}.jpg"
             file_path = osp.join(img_dir_path,img_name)
@@ -59,10 +60,10 @@
             if do_vis:
                 t_bboxes = odb.npchangexyorder(t_bboxes)
                 img = odv.draw_bboxes(img,bboxes=t_bboxes,is_relative_coordinate=False)
-                img = odv.draw_keypoints(img,kps,no_line=False,joints_pair=JOINTS_PAIR)
+                img = odv.draw_keypoints(img,kps,no_line=False,joints_pair=JOINTS_PAIR,left_node=left_node)
                 img = odv.draw_keypoints(img,coco_kps,no_line=True)
                 save_path = osp.join(save_dir,img_name)
                 wmli.imwrite(save_path,img)
             new_coco_data.append([img_name,org_bboxes,kps])
-        with open(coco_pt_path,"wb") as f:
-            pickle.dump(new_coco_data,f)
+        #with open(coco_pt_path,"wb") as f:
+            #pickle.dump(new_coco_data,f)

lib/core/function.py

@@ -11,7 +11,7 @@
 import time
 import logging
 import os
-
+from datadef import *
 import numpy as np
 import torch
 import wtorch.utils as wtu
@@ -90,7 +90,7 @@ def train(config, train_loader, model, criterion, optimizer, epoch,
         batch_time.update(time.time() - end)
         end = time.time()
 
-        if True or i % config.PRINT_FREQ == 0:
+        if i % config.PRINT_FREQ == 0:
             msg = 'Epoch: [{0}][{1}/{2}]\t' \
                   'Time {batch_time.val:.3f}s ({batch_time.avg:.3f}s)\t' \
                   'Speed {speed:.1f} samples/s\t' \
@@ -152,6 +152,7 @@ def validate(config, val_loader, val_dataset, model, criterion, output_dir,
         for i, (input, target, target_weight, meta) in enumerate(val_loader):
             # compute output
             outputs = model(input)
+            #outputs = target.cuda()
             if isinstance(outputs, list):
                 output = outputs[-1]
             else:
@@ -225,6 +226,7 @@ def validate(config, val_loader, val_dataset, model, criterion, output_dir,
                 prefix = '{}_{}'.format(
                     os.path.join(output_dir, 'val'), i
                 )
+                print(f"Save {prefix}")
                 save_debug_images(config, input, meta, target, pred*4, output,
                                   prefix)
 

lib/datadef.py

@@ -24,4 +24,9 @@ def get_ckpt_dir():
 
 def get_log_dir(suffix=""):
     torch_loger_dir = get_data_dir("tmp/tbloger"+suffix)
-    return torch_loger_dir
+    return torch_loger_dir
+
+
+def is_debug(default_value = True):
+    print(f"is_debug {default_value}")
+    return default_value

lib/dataset/JointsDataset.py

@@ -23,6 +23,7 @@
 from utils.transforms import fliplr_joints
 import wtorch.dataset_toolkit as tdt
 import img_utils as wmli
+from datadef import *
 
 
 logger = logging.getLogger(__name__)
@@ -177,6 +178,9 @@ def __getitem__(self, idx):
                 joints, joints_vis = fliplr_joints(
                     joints, joints_vis, data_numpy.shape[1], self.flip_pairs)
                 c[0] = data_numpy.shape[1] - c[0] - 1
+        else:
+            #return self.trans_data_type0(data_numpy,db_rec)
+            pass
 
         trans = get_affine_transform(c, s, r, self.image_size)
         input = cv2.warpAffine(
@@ -232,31 +236,51 @@ def trans_data_type0(self,data_numpy,db_rec):
         bbox = db_rec.get('clean_bbox',None)
         if bbox is None:
             bbox = odk.npget_bbox(joints)
-        bbox = odb.npscale_bboxes(bbox,1.25)
+        bbox = odb.npscale_bboxes(bbox,1.25,max_size=data_numpy.shape[:2][::-1])
         score = db_rec['score'] if 'score' in db_rec else 1
         c = db_rec['center']
-        sf = self.scale_factor
-        rf = self.rotation_factor
-        s =  np.clip(np.random.randn()*sf + 1, 1 - sf, 1 + sf)
-        r = np.clip(np.random.randn()*rf, -rf*2, rf*2) \
-                if random.random() <= 0.6 else 0
 
         #s = 1.4
         #r = 45
         #self.save_vis_kps(data_numpy,joints,"a.jpg",bbox)
-        data_numpy,joints,bbox = odk.rotate(r,data_numpy,joints,bbox,s)
-        joints_vis[:,0] = joints[:,2] 
-        joints_vis[:,1] = joints[:,2] 
-        #self.save_vis_kps(data_numpy,joints,"b.jpg",bbox)
-        if self.flip and random.random() <= 0.5:
-            data_numpy,joints,joints_vis,bbox = odk.flip(data_numpy,joints,joints_vis,self.flip_pairs,
+        if self.is_train:
+            sf = self.scale_factor
+            rf = self.rotation_factor
+            s =  np.clip(np.random.randn()*sf + 1, 1 - sf, 1 + sf)
+            r = np.clip(np.random.randn()*rf, -rf*2, rf*2) \
+                if random.random() <= 0.6 else 0
+            data_numpy,joints,bbox = odk.rotate(r,data_numpy,joints,bbox,s)
+            joints_vis[:,0] = joints[:,2] 
+            joints_vis[:,1] = joints[:,2] 
+
+            #self.save_vis_kps(data_numpy,joints,"b.jpg",bbox)
+            if self.flip and random.random() <= 0.5:
+                data_numpy,joints,joints_vis,bbox = odk.flip(data_numpy,joints,joints_vis,self.flip_pairs,
                                                         bbox=bbox)
+        else:
+            r = 0.0
         #self.save_vis_kps(data_numpy,joints,"b1.jpg",bbox)
         #bbox = odk.npget_bbox(joints)
         #bbox = odb.npscale_bboxes(bbox,1.4)
+        org_img = data_numpy
         data_numpy,bbox = self.cut_and_resize(data_numpy,[bbox],size=self.image_size)
         data_numpy = data_numpy[0]
         bbox = bbox[0]
+        img_a = wmli.sub_imagev2(org_img,bbox.astype(np.int32))
+        img_a = wmli.resize_img(img_a,self.image_size)
+        img_b = data_numpy
+        wmli.imwrite("a.jpg",img_a)
+        wmli.imwrite("b.jpg",img_b)
+
+        c = np.array([(bbox[0]+bbox[2])/2,(bbox[1]+bbox[3])/2],dtype=np.float32)
+        scale = np.array(
+            [
+                (bbox[2]-bbox[0]) * 1.0 / self.pixel_std,
+                (bbox[3]-bbox[1]) * 1.0 / self.pixel_std
+            ],
+            dtype=np.float32
+        )
+
         joints = odk.cut2size(joints,bbox,self.image_size)
         #self.save_vis_kps(data_numpy,joints,"c.jpg",bbox)
 
@@ -276,7 +300,7 @@ def trans_data_type0(self,data_numpy,db_rec):
             'joints': joints,
             'joints_vis': joints_vis,
             'center': c,
-            'scale': db_rec['scale'],
+            'scale': scale,
             'rotation': r,
             'score': score
         }

lib/dataset/coco_mpii.py

@@ -7,7 +7,8 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
+import copy
+from datadef import *
 from collections import defaultdict
 from collections import OrderedDict
 import logging
@@ -63,7 +64,7 @@ def __init__(self, cfg, root, image_set, is_train, transform=None):
         self.lpset_img_dir = "/home/wj/ai/mldata1/lspet/lspet/images"
         self.penn_anno_path = "/home/wj/ai/mldata1/penn_action/Penn_Action/coco_labels"
         self.penn_img_dir = "/home/wj/ai/mldata1/penn_action/Penn_Action/frames"
-        self.crowd_pose_anno_path = '/home/wj/ai/mldata1/crowd_pose/CrowdPose/crowdpose_coco.pt',
+        self.crowd_pose_anno_path = '/home/wj/ai/mldata1/crowd_pose/CrowdPose/crowdpose_coco.pt'
         self.crowd_pose_img_dir =   '/home/wj/ai/mldata1/crowd_pose/images'
 
         self.nms_thre = cfg.TEST.NMS_THRE
@@ -140,7 +141,7 @@ def _load_image_set_index(self):
     def _get_db(self):
         gt_db = []
         if self.is_train:
-            '''if self.is_train or self.use_gt_bbox:
+            if self.is_train or self.use_gt_bbox:
                 # use ground truth bbox
                 gt_db = self._load_coco_keypoint_annotations()
             else:
@@ -149,8 +150,10 @@ def _get_db(self):
 
             gt_db.extend(self._load_mpii_keypoint_annotations())
             gt_db.extend(self._load_lpset_keypoint_annotations())
-            gt_db.extend(self._load_penn_action_keypoint_annotations())'''
+            gt_db.extend(self._load_penn_action_keypoint_annotations())
             gt_db.extend(self._load_crowd_pose_keypoint_annotations())
+            '''if is_debug():
+                gt_db.extend(self._load_penn_action_keypoint_annotations())'''
         else:
             if self.is_train or self.use_gt_bbox:
                 # use ground truth bbox
@@ -212,7 +215,7 @@ def _load_penn_action_keypoint_annotations(self):
             with open(file,"rb") as f:
                 datas = pickle.load(f)
                 for data in datas:
-                    tmp_db = self._load_mpii_keypoint_annotation_kernal(data,img_dir=self.penn_img_dir)
+                    tmp_db = self._load_penn_action_keypoint_annotation_kernal(data,img_dir=self.penn_img_dir)
                     gt_db.extend(tmp_db)
 
         print(f"Total load {len(gt_db)} penn action data.")
@@ -319,8 +322,11 @@ def _load_mpii_keypoint_annotation_kernal(self, data,img_dir):
             joints_3d_vis = np.zeros((self.num_joints, 3), dtype=np.float)
             for ipt in range(self.num_joints):
                 t_vis = kps[ipt,2]
+                if t_vis<0.99 and ipt<5:
+                    t_vis = t_vis*0.5
                 if t_vis > 1:
                     t_vis = 1
+                joints_3d[ipt,2] = t_vis
                 joints_3d_vis[ipt, 0] = t_vis
                 joints_3d_vis[ipt, 1] = t_vis
                 joints_3d_vis[ipt, 2] = 0
@@ -350,6 +356,70 @@ def _load_mpii_keypoint_annotation_kernal(self, data,img_dir):
 
         return rec
 
+    def _load_penn_action_keypoint_annotation_kernal(self, data,img_dir):
+        """
+        coco ann: [u'segmentation', u'area', u'iscrowd', u'image_id', u'bbox', u'category_id', u'id']
+        iscrowd:
+            crowd instances are handled by marking their overlaps with all categories to -1
+            and later excluded in training
+        bbox:
+            [x1, y1, w, h]
+        :param index: coco image id
+        :return: db entry
+        """
+        img_name,bboxes,all_kps = data
+        if bboxes.shape[0]==0:
+            return []
+
+        rec = []
+        for bbox,kps in zip(bboxes,all_kps):
+            # ignore objs without keypoints annotation
+            if np.max(kps[:,:2]) == 0:
+                continue
+
+            joints_3d = kps
+            joints_3d_vis = np.zeros((self.num_joints, 3), dtype=np.float)
+            for ipt in range(self.num_joints):
+                t_vis = kps[ipt,2]
+                if t_vis<0.99 and ipt<5:
+                    t_vis = t_vis*0.5
+                if t_vis > 1:
+                    t_vis = 1
+                joints_3d[ipt,2] = t_vis
+                joints_3d_vis[ipt, 0] = t_vis
+                joints_3d_vis[ipt, 1] = t_vis
+                joints_3d_vis[ipt, 2] = 0
+            src_idxs = list(range(5,17)) 
+            dst_idxs = [6,5,8,7,10,9,12,11,14,13,16,15]
+            tmp_joints_3d = copy.deepcopy(joints_3d)
+            tmp_joints_3d_vis = copy.deepcopy(joints_3d_vis)
+            joints_3d[dst_idxs] = tmp_joints_3d[src_idxs]
+            joints_3d_vis[dst_idxs] = tmp_joints_3d_vis[src_idxs]
+
+            clean_bbox = [bbox[0],bbox[1],bbox[2]-bbox[0],bbox[3]-bbox[1]]
+            if clean_bbox[2]<2 or clean_bbox[3]<2:
+                print(f"Skip {img_dir}/{img_name} {bbox}")
+                continue
+            total_kps_nr = np.sum((kps[...,2]>0.1).astype(np.int32))
+            if total_kps_nr<3:
+                print(f"Skip {img_dir}/{img_name} {total_kps_nr} {kps} {bbox}")
+                continue
+            kp_bbox = odk.npget_bbox(joints_3d)
+            if kp_bbox is not None:
+                bbox = odb.bbox_of_boxes([bbox,kp_bbox])
+            center, scale = self._box2cs(clean_bbox)
+            rec.append({
+                'image': osp.join(img_dir,img_name),
+                'center': center,
+                'scale': scale,
+                'joints_3d': joints_3d,
+                'joints_3d_vis': joints_3d_vis,
+                'clean_bbox':bbox,
+                'filename': img_name,
+                'imgnum': 0,
+            })
+
+        return rec
     def _box2cs(self, box):
         x, y, w, h = box[:4]
         return self._xywh2cs(x, y, w, h)

lib/utils/utils.py

@@ -33,7 +33,10 @@ def create_logger(cfg, cfg_name, phase='train'):
     model = cfg.MODEL.NAME
     cfg_name = os.path.basename(cfg_name).split('.')[0]
 
-    final_output_dir = root_output_dir / dataset / model / cfg_name
+    if phase == 'train':
+        final_output_dir = root_output_dir / dataset / model / cfg_name
+    else:
+        final_output_dir = root_output_dir / dataset / model / (cfg_name+"_"+phase)
 
     print('=> creating {}'.format(final_output_dir))
     final_output_dir.mkdir(parents=True, exist_ok=True)

lib/utils/vis.py

@@ -57,7 +57,7 @@ def save_batch_image_with_joints(batch_image, batch_joints, batch_joints_vis,
                     cv2.circle(ndarr, (int(joint_0), int(joint_1)), 2, color, 2)
                     #cv2.putText(ndarr, f"{k}_{i}", (int(joint_0), int(joint_1)), cv2.FONT_HERSHEY_DUPLEX, 0.5, color, 1)
             k = k + 1
-    cv2.imwrite(file_name, ndarr)
+    cv2.imwrite(file_name, ndarr[...,::-1])
 
 
 def save_batch_heatmaps(batch_image, batch_heatmaps, file_name,
@@ -122,7 +122,7 @@ def save_batch_heatmaps(batch_image, batch_heatmaps, file_name,
 
         grid_image[height_begin:height_end, 0:heatmap_width, :] = resized_image
 
-    cv2.imwrite(file_name, grid_image)
+    cv2.imwrite(file_name, grid_image[...,::-1])
 
 
 def save_debug_images(config, input, meta, target, joints_pred, output,

tools/dist_train.py

@@ -175,7 +175,7 @@ def main():
         final_output_dir, 'checkpoint.pth'
     )
 
-    if cfg.AUTO_RESUME and os.path.exists(checkpoint_file):
+    if False and cfg.AUTO_RESUME and os.path.exists(checkpoint_file):
         logger.info("=> loading checkpoint '{}'".format(checkpoint_file))
         checkpoint = torch.load(checkpoint_file)
         begin_epoch = checkpoint['epoch']
@@ -192,7 +192,6 @@ def main():
             print(f"Load {ckpt_path}.")
             model.module.load_state_dict(torch.load(ckpt_path), strict=True)
 
-            
 
     lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
         optimizer, cfg.TRAIN.LR_STEP, cfg.TRAIN.LR_FACTOR,