feat(translator_commons): add dictionary_exclude to exclude words (#1008)

ksqsf · web-flow · commit 959937e5ed60 · 2025-05-05T15:57:20.000+02:00
Previously, the user must delete words from the dictionary.
diff --git a/src/rime/dict/dictionary.cc b/src/rime/dict/dictionary.cc
@@ -137,6 +137,7 @@ void DictEntryIterator::AddFilter(DictEntryFilter filter) {
   // the introduced filter could invalidate the current or even all the
   // remaining entries
   while (!exhausted() && !filter_(Peek())) {
+    entry_.reset();
     FindNextEntry();
   }
 }
@@ -181,15 +182,12 @@ bool DictEntryIterator::FindNextEntry() {
 }
 
 bool DictEntryIterator::Next() {
-  entry_.reset();
-  if (!FindNextEntry()) {
-    return false;
-  }
-  while (filter_ && !filter_(Peek())) {
+  do {
+    entry_.reset();
     if (!FindNextEntry()) {
       return false;
     }
-  }
+  } while (filter_ && !filter_(Peek()));
   return true;
 }
 
@@ -262,6 +260,7 @@ static void lookup_table(Table* table,
 
 an<DictEntryCollector> Dictionary::Lookup(const SyllableGraph& syllable_graph,
                                           size_t start_pos,
+                                          const hash_set<string>* blacklist,
                                           bool predict_word,
                                           double initial_credibility) {
   if (!loaded())
@@ -275,17 +274,23 @@ an<DictEntryCollector> Dictionary::Lookup(const SyllableGraph& syllable_graph,
   }
   if (collector->empty())
     return nullptr;
-  // sort each group of equal code length
+  // for each group of equal code length, sort it and filter words
   for (auto& v : *collector) {
     v.second.Sort();
+    if (blacklist && !blacklist->empty()) {
+      v.second.AddFilter([blacklist](an<DictEntry> entry) {
+        return entry && !blacklist->count(entry->text);
+      });
+    }
   }
   return collector;
 }
 
 size_t Dictionary::LookupWords(DictEntryIterator* result,
                                const string& str_code,
                                bool predictive,
-                               size_t expand_search_limit) {
+                               size_t expand_search_limit,
+                               const hash_set<string>* blacklist) {
   DLOG(INFO) << "lookup: " << str_code;
   if (!loaded())
     return 0;
@@ -325,6 +330,11 @@ size_t Dictionary::LookupWords(DictEntryIterator* result,
       }
     }
   }
+  if (blacklist && !blacklist->empty()) {
+    result->AddFilter([blacklist](an<DictEntry> entry) {
+      return entry && !blacklist->count(entry->text);
+    });
+  }
   return keys.size();
 }
 
diff --git a/src/rime/dict/dictionary.h b/src/rime/dict/dictionary.h
@@ -71,17 +71,20 @@ class Dictionary : public Class<Dictionary, const Ticket&> {
   RIME_DLL bool Remove();
   RIME_DLL bool Load();
 
-  RIME_DLL an<DictEntryCollector> Lookup(const SyllableGraph& syllable_graph,
-                                         size_t start_pos,
-                                         bool predict_word = false,
-                                         double initial_credibility = 0.0);
+  RIME_DLL an<DictEntryCollector> Lookup(
+      const SyllableGraph& syllable_graph,
+      size_t start_pos,
+      const hash_set<string>* blacklist = nullptr,
+      bool predict_word = false,
+      double initial_credibility = 0.0);
   // if predictive is true, do an expand search with limit,
   // otherwise do an exact match.
   // return num of matching keys.
   RIME_DLL size_t LookupWords(DictEntryIterator* result,
                               const string& str_code,
                               bool predictive,
-                              size_t limit = 0);
+                              size_t limit = 0,
+                              const hash_set<string>* blacklist = nullptr);
   // translate syllable id sequence to string code
   RIME_DLL bool Decode(const Code& code, vector<string>* result);
 
diff --git a/src/rime/gear/reverse_lookup_translator.cc b/src/rime/gear/reverse_lookup_translator.cc
@@ -172,7 +172,7 @@ an<Translation> ReverseLookupTranslator::Query(const string& input,
   bool quality = false;
   if (start < input.length()) {
     if (options_ && options_->enable_completion()) {
-      dict_->LookupWords(&iter, code, true, 100);
+      dict_->LookupWords(&iter, code, true, 100, nullptr);
       quality = !iter.exhausted() && (iter.Peek()->remaining_code_length == 0);
     } else {
       // 2012-04-08 gongchen: fetch multi-syllable words from rev-lookup table
diff --git a/src/rime/gear/script_translator.cc b/src/rime/gear/script_translator.cc
@@ -6,11 +6,11 @@
 //
 // 2011-07-10 GONG Chen <chen.sst@gmail.com>
 //
-#include <algorithm>
 #include <stack>
 #include <cmath>
 #include <boost/algorithm/string/join.hpp>
 #include <boost/range/adaptor/reversed.hpp>
+#include <rime/common.h>
 #include <rime/composition.h>
 #include <rime/candidate.h>
 #include <rime/config.h>
@@ -368,7 +368,8 @@ bool ScriptTranslation::Evaluate(Dictionary* dict, UserDictionary* user_dict) {
   bool predict_word = translator_->enable_word_completion() &&
                       start_ + consumed == end_of_input_;
 
-  phrase_ = dict->Lookup(syllable_graph, 0, predict_word);
+  phrase_ =
+      dict->Lookup(syllable_graph, 0, &translator_->blacklist(), predict_word);
   if (user_dict) {
     const size_t kUnlimitedDepth = 0;
     const size_t kNumSyllablesToPredictWord = 4;
@@ -578,7 +579,8 @@ an<Sentence> ScriptTranslation::MakeSentence(Dictionary* dict,
                                       kMaxSyllablesForUserPhraseQuery));
     }
     // merge lookup results
-    EnrollEntries(same_start_pos, dict->Lookup(syllable_graph, x.first));
+    EnrollEntries(same_start_pos, dict->Lookup(syllable_graph, x.first,
+                                               &translator_->blacklist()));
   }
   if (auto sentence =
           poet_->MakeSentence(graph, syllable_graph.interpreted_length,
diff --git a/src/rime/gear/table_translator.cc b/src/rime/gear/table_translator.cc
@@ -131,6 +131,7 @@ class LazyTableTranslation : public TableTranslation {
 
  private:
   Dictionary* dict_;
+  const hash_set<string>* blacklist_;
   UserDictionary* user_dict_;
   size_t limit_;
   size_t user_dict_limit_;
@@ -150,6 +151,7 @@ LazyTableTranslation::LazyTableTranslation(TableTranslator* translator,
                        end,
                        preedit),
       dict_(translator->dict()),
+      blacklist_(&translator->blacklist()),
       user_dict_(enable_user_dict ? translator->user_dict() : NULL),
       limit_(kInitialSearchLimit),
       user_dict_limit_(kInitialSearchLimit) {
@@ -191,7 +193,7 @@ bool LazyTableTranslation::FetchMoreTableEntries() {
   DLOG(INFO) << "fetching more table entries: limit = " << limit_
              << ", count = " << previous_entry_count;
   DictEntryIterator more;
-  if (dict_->LookupWords(&more, input_, true, limit_) < limit_) {
+  if (dict_->LookupWords(&more, input_, true, limit_, blacklist_) < limit_) {
     DLOG(INFO) << "all table entries obtained.";
     limit_ = 0;  // no more try
   } else {
@@ -263,7 +265,7 @@ an<Translation> TableTranslator::Query(const string& input,
   } else {
     DictEntryIterator iter;
     if (dict_ && dict_->loaded()) {
-      dict_->LookupWords(&iter, code, false);
+      dict_->LookupWords(&iter, code, false, 0, &blacklist());
     }
     UserDictEntryIterator uter;
     if (enable_user_dict) {
@@ -645,7 +647,8 @@ an<Translation> TableTranslator::MakeSentence(const string& input,
         if (homographs.size() >= max_homographs_)
           continue;
         DictEntryIterator iter;
-        dict_->LookupWords(&iter, active_input.substr(0, m.length), false);
+        dict_->LookupWords(&iter, active_input.substr(0, m.length), false, 0,
+                           &blacklist());
         if (filter_by_charset) {
           iter.AddFilter(CharsetFilter::FilterDictEntry);
         }
diff --git a/src/rime/gear/translator_commons.cc b/src/rime/gear/translator_commons.cc
@@ -145,6 +145,16 @@ TranslatorOptions::TranslatorOptions(const Ticket& ticket) {
           tags_.push_back(value->str());
     if (tags_.empty())
       tags_.push_back("abc");
+
+    // blacklist
+    if (auto blacklist =
+            config->GetList(ticket.name_space + "/dictionary_exclude")) {
+      for (auto it = blacklist->begin(); it != blacklist->end(); ++it) {
+        if (auto blackword = As<ConfigValue>(*it)) {
+          blacklist_.insert(blackword->str());
+        }
+      }
+    }
   }
   if (delimiters_.empty()) {
     delimiters_ = " ";
diff --git a/src/rime/gear/translator_commons.h b/src/rime/gear/translator_commons.h
@@ -167,6 +167,7 @@ class TranslatorOptions {
   void set_initial_quality(double quality) { initial_quality_ = quality; }
   Projection& preedit_formatter() { return preedit_formatter_; }
   Projection& comment_formatter() { return comment_formatter_; }
+  const hash_set<string>& blacklist() { return blacklist_; }
 
  protected:
   string delimiters_;
@@ -178,6 +179,7 @@ class TranslatorOptions {
   Projection preedit_formatter_;
   Projection comment_formatter_;
   Patterns user_dict_disabling_patterns_;
+  hash_set<string> blacklist_;
 };
 
 }  // namespace rime

Original file line number	Diff line number	Diff line change
`@@ -137,6 +137,7 @@ void DictEntryIterator::AddFilter(DictEntryFilter filter) {`
`137`	`137`	`// the introduced filter could invalidate the current or even all the`
`138`	`138`	`// remaining entries`
`139`	`139`	`while (!exhausted() && !filter_(Peek())) {`
	`140`	`+ entry_.reset();`
`140`	`141`	`FindNextEntry();`
`141`	`142`	`}`
`142`	`143`	`}`
`@@ -181,15 +182,12 @@ bool DictEntryIterator::FindNextEntry() {`
`181`	`182`	`}`
`182`	`183`
`183`	`184`	`bool DictEntryIterator::Next() {`
`184`		`- entry_.reset();`
`185`		`- if (!FindNextEntry()) {`
`186`		`- return false;`
`187`		`- }`
`188`		`- while (filter_ && !filter_(Peek())) {`
	`185`	`+ do {`
	`186`	`+ entry_.reset();`
`189`	`187`	`if (!FindNextEntry()) {`
`190`	`188`	`return false;`
`191`	`189`	`}`
`192`		`- }`
	`190`	`+ } while (filter_ && !filter_(Peek()));`
`193`	`191`	`return true;`
`194`	`192`	`}`
`195`	`193`
`@@ -262,6 +260,7 @@ static void lookup_table(Table* table,`
`262`	`260`
`263`	`261`	`an<DictEntryCollector> Dictionary::Lookup(const SyllableGraph& syllable_graph,`
`264`	`262`	`size_t start_pos,`
	`263`	`+ const hash_set<string>* blacklist,`
`265`	`264`	`bool predict_word,`
`266`	`265`	`double initial_credibility) {`
`267`	`266`	`if (!loaded())`
`@@ -275,17 +274,23 @@ an<DictEntryCollector> Dictionary::Lookup(const SyllableGraph& syllable_graph,`
`275`	`274`	`}`
`276`	`275`	`if (collector->empty())`
`277`	`276`	`return nullptr;`
`278`		`- // sort each group of equal code length`
	`277`	`+ // for each group of equal code length, sort it and filter words`
`279`	`278`	`for (auto& v : *collector) {`
`280`	`279`	`v.second.Sort();`
	`280`	`+ if (blacklist && !blacklist->empty()) {`
	`281`	`+ v.second.AddFilter([blacklist](an<DictEntry> entry) {`
	`282`	`+ return entry && !blacklist->count(entry->text);`
	`283`	`+ });`
	`284`	`+ }`
`281`	`285`	`}`
`282`	`286`	`return collector;`
`283`	`287`	`}`
`284`	`288`
`285`	`289`	`size_t Dictionary::LookupWords(DictEntryIterator* result,`
`286`	`290`	`const string& str_code,`
`287`	`291`	`bool predictive,`
`288`		`- size_t expand_search_limit) {`
	`292`	`+ size_t expand_search_limit,`
	`293`	`+ const hash_set<string>* blacklist) {`
`289`	`294`	`DLOG(INFO) << "lookup: " << str_code;`
`290`	`295`	`if (!loaded())`
`291`	`296`	`return 0;`
`@@ -325,6 +330,11 @@ size_t Dictionary::LookupWords(DictEntryIterator* result,`
`325`	`330`	`}`
`326`	`331`	`}`
`327`	`332`	`}`
	`333`	`+ if (blacklist && !blacklist->empty()) {`
	`334`	`+ result->AddFilter([blacklist](an<DictEntry> entry) {`
	`335`	`+ return entry && !blacklist->count(entry->text);`
	`336`	`+ });`
	`337`	`+ }`
`328`	`338`	`return keys.size();`
`329`	`339`	`}`
`330`	`340`