README.md

@@ -4,16 +4,18 @@ This repository contains the code and models for the following paper:
 
 
 **[Peeking into the Future: Predicting Future Person Activities and Locations in Videos](https://arxiv.org/abs/1902.03748)** \
-[Junwei Liang](https://www.cs.cmu.edu/~junweil/),
+[Junwei Liang](https://junweiliang.me/),
 [Lu Jiang](http://www.lujiang.info/),
 [Juan Carlos Niebles](http://www.niebles.net/),
 [Alexander Hauptmann](https://www.cs.cmu.edu/~alex/),
 [Li Fei-Fei](http://vision.stanford.edu/feifeili/) \
 [CVPR 2019](http://cvpr2019.thecvf.com/)
 
-You can find more information at our [Project Page](https://next.cs.cmu.edu/).\
+You can find more information at our [Project Page](https://precognition.team/next).\
 *Please note that this is not an officially supported Google product.*
 
++ *[11/2022] CMU server is down. You can replace all `https://next.cs.cmu.edu` with `https://precognition.team/next/` to download necessary resources.*
++ *[02/2020] [New paper](https://precognition.team/next/multiverse/) on multi-future trajectory prediction is accepted by CVPR 2020.*
 
 If you find this code useful in your research then please cite
 
@@ -69,6 +71,7 @@ and locations prediction.
 
 ## Dependencies
 + Python 2.7; TensorFlow == 1.10.0 (Should also work on 1.14+)
++ [10/2020] Now it is compatible with Python 3.6 and Tensorflow 1.15
 
 ## Pretrained Models
 You can download pretrained models by running the script
@@ -88,7 +91,11 @@ Instructions for testing pretrained models can be [found here](TESTING.md).
 Instructions for training new models can be [found here](TRAINING.md).
 
 ## Preparing Rich Visual Features
-Coming soon.
+Instructions for extracting features can be [found here](https://github.com/JunweiLiang/next-prediction/blob/master/code/prepare_data/README.md).
 
 ## Acknowledgments
 The preprecessing code and evaluation code for trajectories were adapted from [Social-GAN](https://github.com/agrimgupta92/sgan).
+
+## PyTorch Implementation
+Checkout this [repo](https://github.com/HRHLALALA/next-prediction-pytorch) based on PyTorch by [HRHLALALA](https://github.com/HRHLALALA).
+

TESTING.md

@@ -2,9 +2,9 @@
 ## Step 1: Prepare the data and model
 We experimented on the [ActEv dataset](https://actev.nist.gov) and the
 [ETH & UCY dataset](https://graphics.cs.ucy.ac.cy/research/downloads/crowd-data).
-The original ActEv annotations can be downloaded from [here](https://next.cs.cmu.edu/data/actev-v1-drop4-yaml.tgz).
+The original ActEv annotations can be downloaded from [here](https://precognition.team/next/data/actev-v1-drop4-yaml.tgz).
 *Please do obtain the data copyright and download the raw videos from their website.*
-You can download our prepared features from the [project page](https://next.cs.cmu.edu)
+You can download our prepared features from the [project page](https://precognition.team/next/)
 by running the script `bash scripts/download_prepared_data.sh`.
 This will download the following data,
 and will require about 31 GB of disk space:

TRAINING.md

@@ -2,9 +2,9 @@
 ## Step 1: Prepare the data and model
 We experimented on the [ActEv dataset](https://actev.nist.gov) and
 the [ETH & UCY dataset](https://graphics.cs.ucy.ac.cy/research/downloads/crowd-data).
-The original ActEv annotations can be downloaded from [here](https://next.cs.cmu.edu/data/actev-v1-drop4-yaml.tgz).
+The original ActEv annotations can be downloaded from [here](https://precognition.team/next/data/actev-v1-drop4-yaml.tgz).
 *Please do obtain the data copyright and download the raw videos from their website.*
-You can download our prepared features from the [project page](next.cs.cmu.edu)
+You can download our prepared features from the [project page](https://precognition.team/next/)
 by running the script `bash scripts/download_prepared_data.sh`.
 This will download the following data, and will require
 about 31 GB of disk space:

code/models.py

@@ -18,6 +18,7 @@
 
 """Model graph definitions and other functions for training and testing."""
 
+import functools
 import math
 import operator
 import os
@@ -203,7 +204,7 @@ def build_forward(self):
     if config.multi_decoder:
       dec_cell_traj = [tf.nn.rnn_cell.LSTMCell(
           config.dec_hidden_size, state_is_tuple=True, name='dec_traj_%s' % i)
-                       for i in xrange(len(config.traj_cats))]
+                       for i in range(len(config.traj_cats))]
       dec_cell_traj = [tf.nn.rnn_cell.DropoutWrapper(
           one, keep_prob) for one in dec_cell_traj]
     else:
@@ -798,7 +799,7 @@ def get_feed_dict(self, batch, is_train=False):
       for j, xy in enumerate(obs_data):
         traj_obs_gt[i, j, :] = xy
         traj_obs_gt_mask[i, j] = True
-      for j in xrange(config.pred_len):
+      for j in range(config.pred_len):
         # used in testing to get the prediction length
         traj_pred_gt_mask[i, j] = True
     # ---------------------------------------
@@ -812,16 +813,16 @@ def get_feed_dict(self, batch, is_train=False):
     feed_dict[self.scene_feat] = data['batch_scene_feat']
 
     # each bacth
-    for i in xrange(len(data['batch_obs_scene'])):
-      for j in xrange(len(data['batch_obs_scene'][i])):
+    for i in range(len(data['batch_obs_scene'])):
+      for j in range(len(data['batch_obs_scene'][i])):
         # it was (1) shaped
         obs_scene[i, j] = data['batch_obs_scene'][i][j][0]
         obs_scene_mask[i, j] = True
 
     # [N,num_scale, T] # each is int to num_grid_class
     for j, _ in enumerate(config.scene_grids):
       this_grid_label = np.zeros([N, T_in], dtype='int32')
-      for i in xrange(len(data['obs_grid_class'])):
+      for i in range(len(data['obs_grid_class'])):
         this_grid_label[i, :] = data['obs_grid_class'][i][j, :]
 
       feed_dict[self.grid_obs_labels[j]] = this_grid_label
@@ -851,8 +852,8 @@ def get_feed_dict(self, batch, is_train=False):
     obs_person_features = np.zeros(
         (N, T_in, person_h, person_w, person_feat_dim), dtype='float32')
 
-    for i in xrange(len(data['obs_boxid'])):
-      for j in xrange(len(data['obs_boxid'][i])):
+    for i in range(len(data['obs_boxid'])):
+      for j in range(len(data['obs_boxid'][i])):
         boxid = data['obs_boxid'][i][j]
         featfile = os.path.join(
             config.person_feat_path, split, '%s.npy' % boxid)
@@ -867,8 +868,8 @@ def get_feed_dict(self, batch, is_train=False):
         (N, T_in, K, config.num_box_class), dtype='float32')
     other_boxes = np.zeros((N, T_in, K, 4), dtype='float32')
     other_boxes_mask = np.zeros((N, T_in, K), dtype='bool')
-    for i in xrange(len(data['obs_other_box'])):
-      for j in xrange(len(data['obs_other_box'][i])):  # -> seq_len
+    for i in range(len(data['obs_other_box'])):
+      for j in range(len(data['obs_other_box'][i])):  # -> seq_len
         this_other_boxes = data['obs_other_box'][i][j]
         this_other_boxes_class = data['obs_other_box_class'][i][j]
 
@@ -911,7 +912,7 @@ def get_feed_dict(self, batch, is_train=False):
 
         this_grid_label = np.zeros([N], dtype='int32')
         this_grid_target = np.zeros([N, 2], dtype='float32')
-        for i in xrange(len(data['pred_grid_class'])):
+        for i in range(len(data['pred_grid_class'])):
           # last pred timestep
           this_grid_label[i] = data['pred_grid_class'][i][j, -1]
           # last pred timestep
@@ -926,7 +927,7 @@ def get_feed_dict(self, batch, is_train=False):
         future_act = np.zeros((N, config.num_act), dtype='uint8')
         # for experiment, training activity detection model
 
-        for i in xrange(len(data['future_activity_onehot'])):
+        for i in range(len(data['future_activity_onehot'])):
           future_act[i, :] = data['future_activity_onehot'][i]
 
         feed_dict[self.future_act_label] = future_act
@@ -935,7 +936,7 @@ def get_feed_dict(self, batch, is_train=False):
     feed_dict[self.traj_class_gt] = np.zeros((N), dtype='int32')
     if config.multi_decoder and is_train:
       traj_class = np.zeros((N), dtype='int32')
-      for i in xrange(len(data['traj_cat'])):
+      for i in range(len(data['traj_cat'])):
         traj_class[i] = data['traj_cat'][i]
       feed_dict[self.traj_class_gt] = traj_class
 
@@ -1034,7 +1035,7 @@ def flatten(tensor, keep):
   # each num in the [] will a*b*c*d...
   # so [0] -> just N here for left
   # for [N, M, JX, di] , left is N*M
-  left = reduce(operator.mul, [fixed_shape[i] or tf.shape(tensor)[i]
+  left = functools.reduce(operator.mul, [fixed_shape[i] or tf.shape(tensor)[i]
                                for i in range(start)])
   # [N, JQ,di]
   # [N*M, JX, di]

code/preprocess.py

@@ -25,10 +25,12 @@
 
 
 import argparse
-import cPickle as pickle
+#import cPickle as pickle
+import pickle
 import glob
 import json
 import os
+import sys
 import numpy as np
 from tqdm import tqdm
 
@@ -104,8 +106,8 @@ def main(args):
     args.scene_grid_centers = []
     for h, w in args.scene_grids:
       h_gap, w_gap = args.video_h*1.0/h, args.video_w*1.0/w
-      centers_x = np.cumsum([w_gap for _ in xrange(w)]) - w_gap/2.0
-      centers_y = np.cumsum([h_gap for _ in xrange(h)]) - h_gap/2.0
+      centers_x = np.cumsum([w_gap for _ in range(w)]) - w_gap/2.0
+      centers_y = np.cumsum([h_gap for _ in range(h)]) - h_gap/2.0
       centers_xx = np.tile(np.expand_dims(centers_x, axis=0), [h, 1])
       centers_yy = np.tile(np.expand_dims(centers_y, axis=1), [1, w])
       centers = np.stack((centers_xx, centers_yy), axis=-1)  # [H,W,2]
@@ -138,7 +140,7 @@ def main(args):
   # For creating the same boxid as previous experiment
   args.person_boxkey2id = None
   if args.person_boxkey2id_p is not None:
-    with open(args.person_boxkey2id_p, "r") as f:
+    with open(args.person_boxkey2id_p, "rb") as f:
       args.person_boxkey2id = pickle.load(f)
 
   prepro_each(args.traj_path, "train", os.path.join(
@@ -223,7 +225,8 @@ def prepro_each(traj_path, split, prepro_path, args):
     scene_oldid2new = {
         int(oldi): scene_oldid2new[oldi] for oldi in scene_oldid2new}
     # for background class or other class that we ignored
-    assert not scene_oldid2new.has_key(0)
+    #assert not scene_oldid2new.has_key(0)
+    assert 0 not in scene_oldid2new
     scene_oldid2new[0] = 0
     total_scene_class = len(scene_oldid2new)
     scene_id2name = scene_id2name["id2name"]
@@ -245,7 +248,17 @@ def prepro_each(traj_path, split, prepro_path, args):
     if args.add_kp:
       kp_file_path = os.path.join(args.kp_path, split, "%s.p" % videoname)
       with open(kp_file_path, "rb") as f:
-        kp_feats = pickle.load(f)
+
+        if sys.version_info.major == 2:
+          # this works for py2 since the pickle is generated with py2 code
+          kp_feats = pickle.load(f)
+        else:
+          # ugly so it is py3 compatitable
+          kp_feats = pickle.load(f, encoding="bytes")
+          new_kp_feats = {}
+          for k in kp_feats:
+            new_kp_feats[k.decode("utf-8")] = kp_feats[k]
+          kp_feats = new_kp_feats
 
     if args.add_scene:
       # get the frameid to file name since scene is not extracted every frames
@@ -690,7 +703,7 @@ def prepro_each(traj_path, split, prepro_path, args):
 
     traj_cat = np.zeros((len(cur_activity)), dtype="uint8")
     count_move = 0
-    for i in xrange(len(cur_activity)):
+    for i in range(len(cur_activity)):
       cur_acts = cur_activity[i]
       move = False
       for actid in cur_acts:
@@ -751,10 +764,11 @@ def prepro_each(traj_path, split, prepro_path, args):
       scene_feat = scene_feat_dict[key]  # [H,W]
       # transform classid first
       new_scene_feat = np.zeros_like(scene_feat)  # zero for background class
-      for i in xrange(scene_h):
-        for j in xrange(scene_w):
+      for i in range(scene_h):
+        for j in range(scene_w):
           # rest is ignored and all put into background
-          if scene_oldid2new.has_key(scene_feat[i, j]):
+          #if scene_oldid2new.has_key(scene_feat[i, j]):
+          if scene_feat[i, j] in scene_oldid2new:
             new_scene_feat[i, j] = scene_oldid2new[scene_feat[i, j]]
       # transform to masks
       this_scene_feat = np.zeros(

code/test.py

@@ -22,10 +22,17 @@
 """
 
 import argparse
-import models
 import tensorflow as tf
+import os
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+# the following will still have colocation debug info
+tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
+
+import models
+
 import utils
 
+
 parser = argparse.ArgumentParser()
 
 # inputs and outputs

code/train.py

@@ -24,8 +24,14 @@
 import argparse
 import math
 import sys
-import models
+
 import tensorflow as tf
+import os
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
+# the following will still have colocation debug info
+tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
+
+import models
 from tqdm import tqdm
 import utils