Add back confidences and adjust default params

Author: Daniel King

scripts/train.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export SEED=13270
+export SEED=15270
 export PYTORCH_SEED=`expr $SEED / 10`
 export NUMPY_SEED=`expr $PYTORCH_SEED / 10`
 
@@ -20,13 +20,13 @@ export WITH_CRF=false  # CRF only works for the baseline
 # training params
 export cuda_device=0
 export BATCH_SIZE=4
-export LR=2e-5
-export TRAINING_DATA_INSTANCES=2000
-export NUM_EPOCHS=4
+export LR=5e-5
+export TRAINING_DATA_INSTANCES=1668
+export NUM_EPOCHS=2
 
 # limit number of sentneces per examples, and number of words per sentence. This is dataset dependant
 export MAX_SENT_PER_EXAMPLE=10
-export SENT_MAX_LEN=40
+export SENT_MAX_LEN=80
 
 # this is for the evaluation of the summarization dataset
 export SCI_SUM=false

sequential_sentence_classification/dataset_reader.py

@@ -75,6 +75,8 @@ def read_one_example(self, json_dict):
         else:
             labels = None
 
+        confidences = json_dict.get("confs", None)
+
         additional_features = None
         if self.sci_sum:
             if self.sci_sum_fake_scores:
@@ -98,18 +100,19 @@ def read_one_example(self, json_dict):
             if len(sentences) == 0:
                 return []
 
-        for sentences_loop, labels_loop, additional_features_loop in  \
-                self.enforce_max_sent_per_example(sentences, labels, additional_features):
+        for sentences_loop, labels_loop, confidences_loop, additional_features_loop in  \
+                self.enforce_max_sent_per_example(sentences, labels, confidences, additional_features):
 
             instance = self.text_to_instance(
                 sentences=sentences_loop,
                 labels=labels_loop,
+                confidences=confidences_loop,
                 additional_features=additional_features_loop,
                 )
             instances.append(instance)
         return instances
 
-    def enforce_max_sent_per_example(self, sentences, labels=None, additional_features=None):
+    def enforce_max_sent_per_example(self, sentences, labels=None, confidences=None, additional_features=None):
         """
         Splits examples with len(sentences) > self.max_sent_per_example into multiple smaller examples
         with len(sentences) <= self.max_sent_per_example.
@@ -121,20 +124,24 @@ def enforce_max_sent_per_example(self, sentences, labels=None, additional_featur
         """
         if labels is not None:
             assert len(sentences) == len(labels)
+        if confidences is not None:
+            assert len(sentences) == len(confidences)
         if additional_features is not None:
             assert len(sentences) == len(additional_features)
 
         if len(sentences) > self.max_sent_per_example and self.max_sent_per_example > 0:
             i = len(sentences) // 2
             l1 = self.enforce_max_sent_per_example(
                     sentences[:i], None if labels is None else labels[:i],
+                    None if confidences is None else confidences[:i],
                     None if additional_features is None else additional_features[:i])
             l2 = self.enforce_max_sent_per_example(
                     sentences[i:], None if labels is None else labels[i:],
+                    None if confidences is None else confidences[i:],
                     None if additional_features is None else additional_features[i:])
             return l1 + l2
         else:
-            return [(sentences, labels, additional_features)]
+            return [(sentences, labels, confidences, additional_features)]
 
     def is_bad_sentence(self, sentence: str):
         if len(sentence) > 10 and len(sentence) < 600:
@@ -171,10 +178,13 @@ def filter_bad_sci_sum_sentences(self, sentences, labels):
     def text_to_instance(self,
                          sentences: List[str],
                          labels: List[str] = None,
+                         confidences: List[float] = None,
                          additional_features: List[float] = None,
                          ) -> Instance:
         if not self.predict:
             assert len(sentences) == len(labels)
+        if confidences is not None:
+            assert len(sentences) == len(confidences)
         if additional_features is not None:
             assert len(sentences) == len(additional_features)
 
@@ -209,6 +219,8 @@ def text_to_instance(self,
                             LabelField(str(label)+"_label") for label in labels
                         ])
 
+        if confidences is not None:
+            fields['confidences'] = ArrayField(np.array(confidences))
         if additional_features is not None:
             fields["additional_features"] = ArrayField(np.array(additional_features))
 

sequential_sentence_classification/model.py

@@ -73,6 +73,7 @@ def __init__(self, vocab: Vocabulary,
     def forward(self,  # type: ignore
                 sentences: torch.LongTensor,
                 labels: torch.IntTensor = None,
+                confidences: torch.Tensor = None,
                 additional_features: torch.Tensor = None,
                 ) -> Dict[str, torch.Tensor]:
         # pylint: disable=arguments-differ
@@ -120,6 +121,9 @@ def forward(self,  # type: ignore
 
                 labels = labels[labels_mask]  # given batch_size x num_sentences_per_example return num_sentences_per_batch
                 assert labels.dim() == 1
+                if confidences is not None:
+                    confidences = confidences[labels_mask]
+                    assert confidences.dim() == 1
                 if additional_features is not None:
                     additional_features = additional_features[labels_mask]
                     assert additional_features.dim() == 2
@@ -132,6 +136,13 @@ def forward(self,  # type: ignore
                                                         # We are ignoring this problem for now.
                                                         # TODO: fix, at least for testing
 
+                # do the same for `confidences`
+                if confidences is not None:
+                    num_confidences = confidences.shape[0]
+                    if num_confidences != num_sentences:
+                        assert num_confidences > num_sentences
+                        confidences = confidences[:num_sentences]
+
                 # and for `additional_features`
                 if additional_features is not None:
                     num_additional_features = additional_features.shape[0]
@@ -141,6 +152,8 @@ def forward(self,  # type: ignore
 
                 # similar to `embedded_sentences`, add an additional dimension that corresponds to batch_size=1
                 labels = labels.unsqueeze(dim=0)
+                if confidences is not None:
+                    confidences = confidences.unsqueeze(dim=0)
                 if additional_features is not None:
                     additional_features = additional_features.unsqueeze(dim=0)
         else:
@@ -185,6 +198,8 @@ def forward(self,  # type: ignore
 
             if not self.with_crf:
                 label_loss = self.loss(flattened_logits.squeeze(), flattened_gold)
+                if confidences is not None:
+                    label_loss = label_loss * confidences.type_as(label_loss).view(-1)
                 label_loss = label_loss.mean()
                 flattened_probs = torch.softmax(flattened_logits, dim=-1)
             else: