We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent ce6d79e commit d198843Copy full SHA for d198843
sequential_sentence_classification/dataset_reader.py
@@ -44,8 +44,6 @@ def __init__(self,
44
predict: bool = False,
45
) -> None:
46
super().__init__(lazy)
47
- # self._word_splitter = word_splitter or SimpleWordSplitter()
48
- # self._tokenizer = tokenizer or WordTokenizer(self._word_splitter)
49
self._tokenizer = WordTokenizer(word_splitter=SpacyWordSplitter(pos_tags=False))
50
self._token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
51
self.sent_max_len = sent_max_len
0 commit comments