fix seq2seq reader bug (#139)

Author: epwalsh

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Fixed BART for latest `transformers` version.
+- Fixed a bug with `Seq2SeqDatasetReader` that would cause an exception when
+  the desired behavior is to not add start or end symbols to either the source or the target
+  and the default `start_symbol` or `end_symbol` are not part of the tokenizer's vocabulary.
 
 ## [v1.1.0](https://github.com/allenai/allennlp-models/releases/tag/v1.1.0) - 2020-09-08
 

allennlp_models/generation/dataset_readers/seq2seq.py

@@ -11,7 +11,7 @@
 from allennlp.data.dataset_readers.dataset_reader import DatasetReader
 from allennlp.data.fields import TextField
 from allennlp.data.instance import Instance
-from allennlp.data.tokenizers import Tokenizer, SpacyTokenizer
+from allennlp.data.tokenizers import Tokenizer, SpacyTokenizer, Token
 from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
 
 logger = logging.getLogger(__name__)
@@ -78,13 +78,29 @@ def __init__(
         self._target_tokenizer = target_tokenizer or self._source_tokenizer
         self._source_token_indexers = source_token_indexers or {"tokens": SingleIdTokenIndexer()}
         self._target_token_indexers = target_token_indexers or self._source_token_indexers
+
         self._source_add_start_token = source_add_start_token
         self._source_add_end_token = source_add_end_token
         self._target_add_start_token = target_add_start_token
         self._target_add_end_token = target_add_end_token
-        self._start_token, self._end_token = self._source_tokenizer.tokenize(
-            start_symbol + " " + end_symbol
-        )
+        self._start_token: Optional[Token] = None
+        self._end_token: Optional[Token] = None
+        if (
+            source_add_start_token
+            or source_add_end_token
+            or target_add_start_token
+            or target_add_end_token
+        ):
+            try:
+                self._start_token, self._end_token = self._source_tokenizer.tokenize(
+                    start_symbol + " " + end_symbol
+                )
+            except ValueError:
+                raise ValueError(
+                    f"Bad start or end symbol ({'start_symbol', 'end_symbol'}) "
+                    f"for tokenizer {self._source_tokenizer}"
+                )
+
         self._delimiter = delimiter
         self._source_max_tokens = source_max_tokens
         self._target_max_tokens = target_max_tokens

tests/generation/dataset_readers/seq2seq_test.py

@@ -216,3 +216,7 @@ def test_correct_quote_handling(self, line):
                 "d",
                 "@end@",
             ]
+
+    def test_bad_start_or_end_symbol(self):
+        with pytest.raises(ValueError, match="Bad start or end symbol"):
+            Seq2SeqDatasetReader(start_symbol="BAD SYMBOL")