# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""Provide Thai word vectors."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, cast
from pythainlp.corpus import get_corpus_path
from pythainlp.tokenize import thai2fit_tokenizer, word_tokenize
if TYPE_CHECKING:
from collections.abc import Callable
import numpy as np
from gensim.models.keyedvectors import Word2VecKeyedVectors
from numpy.typing import NDArray
WV_DIM: int = 300 # word vector dimension
_MODEL_NAME: str = "thai2fit_wv"
_TK_SP: str = "xxspace"
_TK_EOL: str = "xxeol"
class _DuplicateWordFilter(logging.Filter):
"""Suppress gensim's 'duplicate word' warnings for word2vec files."""
def filter(self, record: logging.LogRecord) -> bool:
return "duplicate word" not in record.getMessage()
[docs]
class WordVector:
"""
Provide Thai word vectors.
:param str model_name: name of the word vector model
* *thai2fit_wv* - word vectors from thai2fit (default)
* *ltw2v* - word vectors from LTW2V: The Large Thai Word2Vec v0.1
* *ltw2v_v1.0_15_window* - word vectors from LTW2V v1.0 with a
window size of 15
* *ltw2v_v1.0_5_window* - word vectors from LTW2V v1.0 with a
window size of 5
"""
[docs]
def __init__(self, model_name: str = "thai2fit_wv") -> None:
"""
Initialize the word vector.
:param str model_name: name of the word vector model
"""
self.model_name: str
self.model: Word2VecKeyedVectors
self.WV_DIM: int
self.tokenize: Callable[[str], list[str]]
self.load_wordvector(model_name)
[docs]
def load_wordvector(self, model_name: str) -> None:
"""
Load a word vector model.
:param str model_name: name of the word vector model
:raises FileNotFoundError: if the model corpus is not found
"""
from gensim.models import KeyedVectors
self.model_name = model_name
corpus_file = get_corpus_path(self.model_name)
if not corpus_file:
raise FileNotFoundError(
f"corpus-not-found name={model_name!r}\n"
f" Corpus '{model_name}' not found.\n"
f" Python: pythainlp.corpus.download('{model_name}')\n"
f" CLI: thainlp data get {model_name}"
)
_filter = _DuplicateWordFilter()
_gensim_kv_logger = logging.getLogger("gensim.models.keyedvectors")
_gensim_kv_logger.addFilter(_filter)
try:
self.model = KeyedVectors.load_word2vec_format(
corpus_file,
binary=True,
unicode_errors="ignore",
)
finally:
_gensim_kv_logger.removeFilter(_filter)
self.WV_DIM = self.model.vector_size
if self.model_name == "thai2fit_wv":
self.tokenize = thai2fit_tokenizer().word_tokenize
else:
self.tokenize = word_tokenize
[docs]
def get_model(self) -> Word2VecKeyedVectors:
"""
Return the word vector model.
:return: `gensim` word2vec model
:rtype: gensim.models.keyedvectors.Word2VecKeyedVectors
"""
return self.model
[docs]
def doesnt_match(self, words: list[str]) -> str:
"""
Return the word that is the most unrelated to other words.
Use the function :func:`doesnt_match` from :mod:`gensim`.
:param list[str] words: list of words
:return: the most unrelated word
:rtype: str
:raises KeyError: if a word in **words** is not in the vocabulary
of the model
:Note:
* If a word in `words` is not in the vocabulary, :class:`KeyError`
will be raised.
:Example:
Pick the word "พริกไทย" (name of food) out of the list of meals
("อาหารเช้า", "อาหารเที่ยง", "อาหารเย็น").
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> words = ["อาหารเช้า", "อาหารเที่ยง", "อาหารเย็น", "พริกไทย"]
>>> wv.doesnt_match(words)
'พริกไทย'
Pick the word "เรือ" (name of vehicle) out of the list of words
related to occupation ("ดีไซน์เนอร์", "พนักงานเงินเดือน", "หมอ").
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> words = ["ดีไซน์เนอร์", "พนักงานเงินเดือน", "หมอ", "เรือ"]
>>> wv.doesnt_match(words)
'เรือ'
"""
return cast("str", self.model.doesnt_match(words))
[docs]
def most_similar_cosmul(
self, positive: list[str], negative: list[str]
) -> list[tuple[str, float]]:
"""
Find the top-10 words most similar to two lists of words.
The two lists are labeled as positive and negative. The top-10
most similar words are obtained using the multiplication
combination objective from Omer Levy and Yoav Goldberg
[OmerLevy_YoavGoldberg_2014]_.
Use the function :func:`gensim.most_similar_cosmul` directly from
:mod:`gensim`.
:param list[str] positive: list of words to add
:param list[str] negative: list of words to subtract
:return: list of the top-10 most similar words and their
similarity scores
:rtype: list[tuple[str, float]]
:raises KeyError: if a word in **positive** or **negative** is not
in the vocabulary of the model
:Note:
* With a single word in the positive list, it finds the
most similar words to the given word (similar
to :func:`gensim.most_similar`)
* If a word in `positive` or `negative` is not in the vocabulary,
:class:`KeyError` will be raised.
:Example:
Find the **top-10** most similar words to the word: "แม่น้ำ".
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> list_positive = ["แม่น้ำ"]
>>> list_negative = []
>>> wv.most_similar_cosmul(list_positive, list_negative)
[('ลำน้ำ', 0.8206598162651062), ('ทะเลสาบ', 0.775945782661438),
('ลุ่มน้ำ', 0.7490593194961548), ('คลอง', 0.7471904754638672),
('ปากแม่น้ำ', 0.7354257106781006), ('ฝั่งแม่น้ำ', 0.7120099067687988),
('ทะเล', 0.7030453681945801), ('ริมแม่น้ำ', 0.7015200257301331),
('แหล่งน้ำ', 0.6997432112693787), ('ภูเขา', 0.6960948705673218)]
Find the **top-10** most similar words to the words: "นายก",
"รัฐมนตรี", and "ประเทศ".
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> list_positive = ["นายก", "รัฐมนตรี", "ประเทศ"]
>>> list_negative = []
>>> wv.most_similar_cosmul(list_positive, list_negative)
[('รองนายกรัฐมนตรี', 0.2730445861816406),
('เอกอัครราชทูต', 0.26500266790390015),
('นายกรัฐมนตรี', 0.2649088203907013),
('ผู้ว่าราชการจังหวัด', 0.25119125843048096),
('ผู้ว่าการ', 0.2510434687137604), ('เลขาธิการ', 0.24824175238609314),
('ผู้ว่า', 0.2453523576259613), ('ประธานกรรมการ', 0.24147476255893707),
('รองประธาน', 0.24123257398605347), ('สมาชิกวุฒิสภา',
0.2405330240726471)]
Find the **top-10** most similar words when having **only** positive
list and **both** positive and negative lists.
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> list_positive = ["ประเทศ", "ไทย", "จีน", "ญี่ปุ่น"]
>>> list_negative = []
>>> wv.most_similar_cosmul(list_positive, list_negative)
[('ประเทศจีน', 0.22022421658039093), ('เกาหลี', 0.2196873426437378),
('สหรัฐอเมริกา', 0.21660110354423523),
('ประเทศญี่ปุ่น', 0.21205860376358032),
('ประเทศไทย', 0.21159221231937408), ('เกาหลีใต้',
0.20321202278137207),
('อังกฤษ', 0.19610872864723206), ('ฮ่องกง', 0.1928885132074356),
('ฝรั่งเศส', 0.18383873999118805), ('พม่า', 0.18369348347187042)]
>>>
>>> list_positive = ["ประเทศ", "ไทย", "จีน", "ญี่ปุ่น"]
>>> list_negative = ["อเมริกา"]
>>> wv.most_similar_cosmul(list_positive, list_negative)
[('ประเทศไทย', 0.3278159201145172), ('เกาหลี', 0.3201899230480194),
('ประเทศจีน', 0.31755179166793823), ('พม่า', 0.30845439434051514),
('ประเทศญี่ปุ่น', 0.306713730096817),
('เกาหลีใต้', 0.3003999888896942),
('ลาว', 0.2995176911354065), ('คนไทย', 0.2885020673274994),
('เวียดนาม', 0.2878379821777344), ('ชาวไทย', 0.28480708599090576)]
The function raises :class:`KeyError` when the term "เมนูอาหารไทย"
is not in the vocabulary.
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> list_positive = ["เมนูอาหารไทย"]
>>> list_negative = []
>>> wv.most_similar_cosmul(list_positive, list_negative)
KeyError: "word 'เมนูอาหารไทย' not in vocabulary"
"""
return cast(
"list[tuple[str, float]]",
self.model.most_similar_cosmul(
positive=positive, negative=negative
),
)
[docs]
def similarity(self, word1: str, word2: str) -> float:
"""
Compute the cosine similarity between two words.
:param str word1: first word to be compared
:param str word2: second word to be compared
:return: cosine similarity between the two word vectors
:rtype: float
:raises KeyError: if **word1** or **word2** is not in the
vocabulary of the model
:Note:
* If a word in `word1` or `word2` is not in the vocabulary,
:class:`KeyError` will be raised.
:Example:
Compute cosine similarity between two words: "รถไฟ" and "รถไฟฟ้า"
(train and electric train).
>>> from pythainlp.word_vector import WordVector
>>> wv = WordVector()
>>> wv.similarity("รถไฟ", "รถไฟฟ้า")
0.43387136
Compute cosine similarity between two words: "เสือดาว" and "รถไฟฟ้า"
(leopard and electric train).
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> wv.similarity("เสือดาว", "รถไฟฟ้า")
0.04300258
"""
return cast("float", self.model.similarity(word1, word2))
[docs]
def sentence_vectorizer(
self, text: str, use_mean: bool = True
) -> NDArray[np.float32]:
"""
Convert Thai text into a vector.
First tokenize the text and map each word to its word vector
from the model. Then aggregate the word vectors into one vector
of 300 dimensions, by the mean or the sum of all word vectors.
:param str text: text to be vectorized
:param bool use_mean: if True, aggregate word vectors by their
mean; otherwise, aggregate by their sum
:return: :class:`numpy.ndarray` of dtype ``numpy.float32`` and
shape ``(1, 300)`` representing the given text
:rtype: numpy.typing.NDArray[numpy.float32]
:Example:
Vectorize the sentence, "อ้วนเสี้ยวเข้ายึดแคว้นกิจิ๋ว ในปี พ.ศ. 735",
into one sentence vector with two aggregation methods: mean
and summation.
>>> from pythainlp.word_vector import WordVector
>>>
>>> wv = WordVector()
>>> sentence = "อ้วนเสี้ยวเข้ายึดแคว้นกิจิ๋ว ในปี พ.ศ. 735"
>>> wv.sentence_vectorizer(sentence, use_mean=True)
array([[-0.00421414, -0.08881307, 0.05081136, -0.05632929,
-0.06607185, 0.03059357, -0.113882 , -0.00074836, 0.05035743,
0.02914307,
...
0.02893357, 0.11327957, 0.04562086, -0.05015393, 0.11641257,
0.32304936, -0.05054322, 0.03639471, -0.06531371, 0.05048079]])
>>>
>>> wv.sentence_vectorizer(sentence, use_mean=False)
array([[-0.05899798, -1.24338295, 0.711359 , -0.78861002,
-0.92500597, 0.42831 , -1.59434797, -0.01047703, 0.705004
, 0.40800299,
...
0.40506999, 1.58591403, 0.63869202, -0.702155 , 1.62977601,
4.52269109, -0.70760502, 0.50952601, -0.914392 , 0.70673105]])
"""
import numpy as np
vec = np.zeros((1, self.WV_DIM), dtype=np.float32)
words = self.tokenize(text)
len_words = len(words)
if not len_words:
return vec
for word in words:
token = word
if word == " " and self.model_name == "thai2fit_wv":
token = _TK_SP
elif word == "\n" and self.model_name == "thai2fit_wv":
token = _TK_EOL
if token in self.model.index_to_key:
vec += self.model.get_vector(token)
if use_mean:
vec /= len_words
return vec