init

Author: CalciferZh

.gitignore

@@ -1,3 +1,5 @@
+model/mano_v1_2
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1 +1,56 @@
-# minimal-hand
+# Minimal Hand
+
+A minimal solution to hand motion capture from a single color camera at over 100fps.
+Easy to use, plug to run.
+
+![teaser](teaser.gif)
+
+This is the official implementation of the paper "Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data" (CVPR 2020).
+
+This project provides the core components for hand motion capture:
+1. estimating joint **locations** from a monocular RGB image (DetNet)
+1. estimating joint **rotations** from locations (IKNet)
+
+We focus on:
+1. ease of use (all you need is a webcam)
+1. time efficiency (on our 1080Ti, 8.9ms for DetNet, 0.9ms for IKNet)
+1. robustness to occlusion, hand-object interaction, fast motion, changing scale and view point
+
+## Usage
+
+### Install dependencies
+Please check `requirements.txt`. All dependencies are available via pip and conda.
+
+### Prepare MANO hand model
+1. Download MANO model from [here](https://mano.is.tue.mpg.de/) and unzip it.
+1. In `config.py`, set `OFFICIAL_MANO_PATH` to the **left hand** model.
+1. Run `python prepare_mano.py`, you will get the converted MANO model that is compatible with this project at `config.HAND_MESH_MODEL_PATH`.
+
+### Prepare pre-trained network models
+1. Download models from [here](https://drive.google.com/open?id=1ZnDYF9rHKbef27tiHkWrLznqe18le7ol).
+1. Put `detnet.ckpt.*` in `model/detnet`, and `iknet.ckpt.*` in `model/iknet`.
+1. Check `config.py`, make sure all required files are there.
+
+### Run the demo for webcam input
+1. `python app.py`
+1. Put your **right hand** in front of the camera. The pre-trained model is for left hand, but the input would be flipped internally.
+1. Press `ESC` to quit.
+
+### Use the models in your project
+Please check `wrappers.py`.
+
+## Citation
+
+If you find the project helpful, please consider citing us:
+```
+@inproceedings{zhou2019monocular,
+  title={Monocular Real-time Hand Shape and Motion Capture using Multi-modal Data},
+  author={Zhou, Yuxiao and Habermann, Marc and Xu, Weipeng and Habibie, Ikhsanul and Theobalt, Christian and Xu, Feng},
+  booktitle={Proceedings of the IEEE International Conference on Computer Vision},
+  pages={0--0},
+  year={2020}
+}
+```
+
+## IKNet Alternative
+We also provide an optimization-based IK solver [here](https://github.com/CalciferZh/Minimal-IK).

app.py

@@ -0,0 +1,121 @@
+import cv2
+import keyboard
+import numpy as np
+import open3d as o3d
+import pygame
+from transforms3d.axangles import axangle2mat
+
+import config
+from capture import OpenCVCapture
+from hand_mesh import HandMesh
+from kinematics import mpii_to_mano
+from utils import OneEuroFilter, imresize
+from wrappers import ModelPipeline
+from utils import *
+
+
+def live_application(capture):
+  """
+  Launch an application that reads from a webcam and estimates hand pose at
+  real-time.
+
+  The captured hand must be the right hand, but will be flipped internally
+  and rendered.
+
+  Parameters
+  ----------
+  capture : object
+    An object from `capture.py` to read capture stream from.
+  """
+  ############ output visualization ############
+  view_mat = axangle2mat([1, 0, 0], np.pi) # align different coordinate systems
+  window_size = 1080
+
+  hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
+  mesh = o3d.geometry.TriangleMesh()
+  mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
+  mesh.vertices = \
+    o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
+  mesh.compute_vertex_normals()
+
+  viewer = o3d.visualization.Visualizer()
+  viewer.create_window(
+    width=window_size + 1, height=window_size + 1,
+    window_name='Minimal Hand - output'
+  )
+  viewer.add_geometry(mesh)
+
+  view_control = viewer.get_view_control()
+  cam_params = view_control.convert_to_pinhole_camera_parameters()
+  extrinsic = cam_params.extrinsic.copy()
+  extrinsic[0:3, 3] = 0
+  cam_params.extrinsic = extrinsic
+  cam_params.intrinsic.set_intrinsics(
+    window_size + 1, window_size + 1, config.CAM_FX, config.CAM_FY,
+    window_size // 2, window_size // 2
+  )
+  view_control.convert_from_pinhole_camera_parameters(cam_params)
+  view_control.set_constant_z_far(1000)
+
+  render_option = viewer.get_render_option()
+  render_option.load_from_json('./render_option.json')
+  viewer.update_renderer()
+
+  ############ input visualization ############
+  pygame.init()
+  display = pygame.display.set_mode((window_size, window_size))
+  pygame.display.set_caption('Minimal Hand - input')
+
+  ############ misc ############
+  mesh_smoother = OneEuroFilter(4.0, 0.0)
+  clock = pygame.time.Clock()
+  model = ModelPipeline()
+
+  while True:
+    frame_large = capture.read()
+    if frame_large is None:
+      continue
+    if frame_large.shape[0] > frame_large.shape[1]:
+      margin = int((frame_large.shape[0] - frame_large.shape[1]) / 2)
+      frame_large = frame_large[margin:-margin]
+    else:
+      margin = int((frame_large.shape[1] - frame_large.shape[0]) / 2)
+      frame_large = frame_large[:, margin:-margin]
+
+    frame_large = np.flip(frame_large, axis=1).copy()
+    frame = imresize(frame_large, (128, 128))
+
+    _, theta_mpii = model.process(frame)
+    theta_mano = mpii_to_mano(theta_mpii)
+
+    v = hand_mesh.set_abs_quat(theta_mano)
+    v *= 2 # for better visualization
+    v = v * 1000 + np.array([0, 0, 400])
+    v = mesh_smoother.process(v)
+    mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
+    mesh.vertices = o3d.utility.Vector3dVector(np.matmul(view_mat, v.T).T)
+    mesh.paint_uniform_color(config.HAND_COLOR)
+    mesh.compute_triangle_normals()
+    mesh.compute_vertex_normals()
+    viewer.update_geometry()
+
+    viewer.poll_events()
+
+    display.blit(
+      pygame.surfarray.make_surface(
+        np.transpose(
+          imresize(frame_large, (window_size, window_size)
+        ), (1, 0, 2))
+      ),
+      (0, 0)
+    )
+    pygame.display.update()
+
+    if keyboard.is_pressed("esc"):
+      break
+
+    clock.tick(30)
+
+
+if __name__ == '__main__':
+  live_application(OpenCVCapture())

capture.py

@@ -0,0 +1,27 @@
+import cv2
+import numpy as np
+
+
+class OpenCVCapture:
+  """
+  OpenCV wrapper to read from webcam.
+  """
+  def __init__(self):
+    """
+    Init.
+    """
+    self.cap = cv2.VideoCapture(0)
+
+  def read(self):
+    """
+    Read one frame. Note this function might be blocked by the sensor.
+
+    Returns
+    -------
+    np.ndarray
+      Read frame. Might be `None` is the webcam fails to get on frame.
+    """
+    flag, frame = self.cap.read()
+    if not flag:
+      return None
+    return np.flip(frame, -1).copy() # BGR to RGB

config.py

@@ -0,0 +1,12 @@
+DETECTION_MODEL_PATH = './model/detnet/detnet.ckpt'
+IK_MODEL_PATH = './model/iknet/iknet.ckpt'
+HAND_MESH_MODEL_PATH = './model/hand_mesh/hand_mesh_model.pkl'
+# use left hand
+OFFICIAL_MANO_PATH = './model/mano_v1_2/models/MANO_LEFT.pkl'
+IK_UNIT_LENGTH = 0.09473151311686484 # in meter
+
+HAND_COLOR = [228/255, 178/255, 148/255]
+
+# only for rendering
+CAM_FX = 620.744
+CAM_FY = 621.151

hand_mesh.py

@@ -0,0 +1,83 @@
+import numpy as np
+from transforms3d.quaternions import quat2mat
+
+from config import *
+from kinematics import *
+from utils import *
+
+
+class HandMesh():
+  """
+  Wrapper for the MANO hand model.
+  """
+  def __init__(self, model_path):
+    """
+    Init.
+
+    Parameters
+    ----------
+    model_path : str
+      Path to the MANO model file. This model is converted by `prepare_mano.py`
+      from official release.
+    """
+    params = load_pkl(model_path)
+    self.verts = params['verts']
+    self.faces = params['faces']
+    self.weights = params['weights']
+    self.joints = params['joints']
+
+    self.n_verts = self.verts.shape[0]
+    self.n_faces = self.faces.shape[0]
+
+    self.ref_pose = []
+    self.ref_T = []
+    for j in range(MANOHandJoints.n_joints):
+      parent = MANOHandJoints.parents[j]
+      if parent is None:
+        self.ref_T.append(self.verts)
+        self.ref_pose.append(self.joints[j])
+      else:
+        self.ref_T.append(self.verts - self.joints[parent])
+        self.ref_pose.append(self.joints[j] - self.joints[parent])
+    self.ref_pose = np.expand_dims(np.stack(self.ref_pose, 0), -1)
+    self.ref_T = np.expand_dims(np.stack(self.ref_T, 1), -1)
+
+  def set_abs_quat(self, quat):
+    """
+    Set absolute (global) rotation for the hand.
+
+    Parameters
+    ----------
+    quat : np.ndarray, shape [J, 4]
+      Absolute rotations for each joint in quaternion.
+
+    Returns
+    -------
+    np.ndarray, shape [V, 3]
+      Mesh vertices after posing.
+    """
+    mats = []
+    for j in range(MANOHandJoints.n_joints):
+      mats.append(quat2mat(quat[j]))
+    mats = np.stack(mats, 0)
+
+    pose = np.matmul(mats, self.ref_pose)
+    joint_xyz = [None] * MANOHandJoints.n_joints
+    for j in range(MANOHandJoints.n_joints):
+      joint_xyz[j] = pose[j]
+      parent = MANOHandJoints.parents[j]
+      if parent is not None:
+        joint_xyz[j] += joint_xyz[parent]
+    joint_xyz = np.stack(joint_xyz, 0)[..., 0]
+
+    T = np.matmul(np.expand_dims(mats, 0), self.ref_T)[..., 0]
+    self.verts = [None] * MANOHandJoints.n_joints
+    for j in range(MANOHandJoints.n_joints):
+      self.verts[j] = T[:, j]
+      parent = MANOHandJoints.parents[j]
+      if parent is not None:
+        self.verts[j] += joint_xyz[parent]
+    self.verts = np.stack(self.verts, 1)
+    self.verts = np.sum(self.verts * self.weights, 1)
+
+    return self.verts.copy()