Merge pull request #1316 from PyThaiNLP/copilot/fix-duplicate-word-warning

fix: suppress gensim duplicate-word warnings when loading word2vec binary files

Author: bact

CHANGELOG.md

@@ -57,7 +57,8 @@ The minimum requirement is now Python 3.9.
 ### Deprecated
 
 - `PYTHAINLP_DATA_DIR` env var; use `PYTHAINLP_DATA` instead
-  (follows `NLTK_DATA` convention from NLTK) (#1306)
+  (follows `NLTK_DATA` convention from NLTK)
+  `PYTHAINLP_DATA_DIR` will be removed in future version (#1306)
 
 ### Removed
 
@@ -72,6 +73,8 @@ The minimum requirement is now Python 3.9.
 - Consonant cluster boundary issue in `royin` romanization (#1172)
 - Final consonant classification in `check_marttra()` (#1173)
 - Kho Khon alphabet issue in `tltk` transliteration (#1187)
+- Suppress Gensim duplicate-word warnings when loading word2vec
+  binary files (#1316)
 
 ## [5.2.0] - 2025-12-20
 

pythainlp/augment/word2vec/core.py

@@ -4,12 +4,20 @@
 from __future__ import annotations
 
 import itertools
+import logging
 from typing import TYPE_CHECKING, Callable, Union
 
 if TYPE_CHECKING:
     from gensim.models.keyedvectors import KeyedVectors
 
 
+class _DuplicateWordFilter(logging.Filter):
+    """Suppress gensim's 'duplicate word' warnings for word2vec files."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return "duplicate word" not in record.getMessage()
+
+
 class Word2VecAug:
     tokenizer: Callable[[str], list[str]]
     model: "KeyedVectors"
@@ -28,12 +36,19 @@ def __init__(
         import gensim.models.keyedvectors as word2vec
 
         self.tokenizer: Callable[[str], list[str]] = tokenize
-        if type == "file":
-            self.model = word2vec.KeyedVectors.load_word2vec_format(model)
-        elif type == "binary":
-            self.model = word2vec.KeyedVectors.load_word2vec_format(
-                model, binary=True, unicode_errors="ignore"
-            )
+        if type in ("file", "binary"):
+            _filter = _DuplicateWordFilter()
+            _gensim_kv_logger = logging.getLogger("gensim.models.keyedvectors")
+            _gensim_kv_logger.addFilter(_filter)
+            try:
+                if type == "file":
+                    self.model = word2vec.KeyedVectors.load_word2vec_format(model)
+                else:
+                    self.model = word2vec.KeyedVectors.load_word2vec_format(
+                        model, binary=True, unicode_errors="ignore"
+                    )
+            finally:
+                _gensim_kv_logger.removeFilter(_filter)
         else:
             self.model = model
         self.dict_wv: list[str] = list(self.model.key_to_index.keys())

pythainlp/word_vector/core.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
+import logging
 from collections.abc import Callable
 from typing import TYPE_CHECKING
 
@@ -21,6 +22,13 @@
 _TK_EOL: str = "xxeol"
 
 
+class _DuplicateWordFilter(logging.Filter):
+    """Suppress gensim's 'duplicate word' warnings for word2vec files."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return "duplicate word" not in record.getMessage()
+
+
 class WordVector:
     """Word Vector class
 
@@ -66,11 +74,17 @@ def load_wordvector(self, model_name: str) -> None:
                 f"    Python: pythainlp.corpus.download('{model_name}')\n"
                 f"    CLI:    thainlp data get {model_name}"
             )
-        self.model = KeyedVectors.load_word2vec_format(
-            corpus_file,
-            binary=True,
-            unicode_errors="ignore",
-        )
+        _filter = _DuplicateWordFilter()
+        _gensim_kv_logger = logging.getLogger("gensim.models.keyedvectors")
+        _gensim_kv_logger.addFilter(_filter)
+        try:
+            self.model = KeyedVectors.load_word2vec_format(
+                corpus_file,
+                binary=True,
+                unicode_errors="ignore",
+            )
+        finally:
+            _gensim_kv_logger.removeFilter(_filter)
         self.WV_DIM = self.model.vector_size
 
         if self.model_name == "thai2fit_wv":