Adding probability of masking a token parameter for LM task (#630)

Author: pashok3d

farm/data_handler/input_features.py

@@ -245,7 +245,7 @@ def samples_to_features_ner(
     return [feature_dict]
 
 
-def samples_to_features_bert_lm(sample, max_seq_len, tokenizer, next_sent_pred=True):
+def samples_to_features_bert_lm(sample, max_seq_len, tokenizer, next_sent_pred=True, masked_lm_prob=0.15):
     """
     Convert a raw sample (pair of sentences as tokenized strings) into a proper training sample with
     IDs, LM labels, padding_mask, CLS and SEP tokens etc.
@@ -255,6 +255,8 @@ def samples_to_features_bert_lm(sample, max_seq_len, tokenizer, next_sent_pred=T
     :param max_seq_len: Maximum length of sequence.
     :type max_seq_len: int
     :param tokenizer: Tokenizer
+    :param masked_lm_prob: probability of masking a token
+    :type masked_lm_prob: float
     :return: InputFeatures, containing all inputs and labels of one sample as IDs (as used for model training)
     """
 
@@ -264,10 +266,10 @@ def samples_to_features_bert_lm(sample, max_seq_len, tokenizer, next_sent_pred=T
 
         # mask random words
         tokens_a, t1_label = mask_random_words(tokens_a, tokenizer.vocab,
-                                               token_groups=sample.tokenized["text_a"]["start_of_word"])
+                                               token_groups=sample.tokenized["text_a"]["start_of_word"], masked_lm_prob=masked_lm_prob)
 
         tokens_b, t2_label = mask_random_words(tokens_b, tokenizer.vocab,
-                                               token_groups=sample.tokenized["text_b"]["start_of_word"])
+                                               token_groups=sample.tokenized["text_b"]["start_of_word"], masked_lm_prob=masked_lm_prob)
 
         if tokenizer.is_fast:
             # Detokenize input as fast tokenizer can't handle tokenized input
@@ -290,7 +292,7 @@ def samples_to_features_bert_lm(sample, max_seq_len, tokenizer, next_sent_pred=T
         tokens_a = sample.tokenized["text_a"]["tokens"]
         tokens_b = None
         tokens_a, t1_label = mask_random_words(tokens_a, tokenizer.vocab,
-                                               token_groups=sample.tokenized["text_a"]["start_of_word"])
+                                               token_groups=sample.tokenized["text_a"]["start_of_word"], masked_lm_prob=masked_lm_prob)
         if tokenizer.is_fast:
             # Detokenize input as fast tokenizer can't handle tokenized input
             tokens_a = " ".join(tokens_a)

farm/data_handler/processor.py

@@ -845,6 +845,8 @@ def __init__(
         next_sent_pred_style="sentence",
         max_docs=None,
         proxies=None,
+        masked_lm_prob=0.15,
+        
         **kwargs
     ):
         """
@@ -885,6 +887,8 @@ def __init__(
         :param proxies: proxy configuration to allow downloads of remote datasets.
                         Format as in  "requests" library: https://2.python-requests.org//en/latest/user/advanced/#proxies
         :type proxies: dict
+        :param masked_lm_prob: probability of masking a token
+        :type masked_lm_prob: float
         :param kwargs: placeholder for passing generic parameters
         :type kwargs: object
         """
@@ -910,6 +914,8 @@ def __init__(
         self.add_task("lm", "acc", list(self.tokenizer.vocab) + added_tokens)
         if self.next_sent_pred:
             self.add_task("nextsentence", "acc", ["False", "True"])
+        self.masked_lm_prob = masked_lm_prob
+
 
     def get_added_tokens(self):
         dictionary = self.tokenizer.added_tokens_encoder
@@ -1064,7 +1070,7 @@ def _dict_to_samples_no_next_sent(self, doc):
     def _sample_to_features(self, sample) -> dict:
         features = samples_to_features_bert_lm(
             sample=sample, max_seq_len=self.max_seq_len, tokenizer=self.tokenizer,
-            next_sent_pred=self.next_sent_pred
+            next_sent_pred=self.next_sent_pred, masked_lm_prob=self.masked_lm_prob
         )
         return features
 
@@ -1205,7 +1211,6 @@ def __init__(
             tasks={},
             proxies=proxies
         )
-
         if metric and label_list:
             self.add_task("question_answering", metric, label_list)
         else:

test/test_lm_finetuning.py

@@ -251,4 +251,4 @@ def test_lm_finetuning_custom_vocab(caplog):
     assert isinstance(result[0]["vec"][0], np.float32)
 
 if(__name__=="__main__"):
-    test_lm_finetuning()
+    test_lm_finetuning()