# SPDX-FileCopyrightText: 2016-2026 PyThaiNLP Project
# SPDX-FileType: SOURCE
# SPDX-License-Identifier: Apache-2.0
"""
Tokenize Thai text into words with multi-cut, a maximum matching approach.
The original code is by Korakot Chaovavanich.
:See Also:
* Facebook post:
https://www.facebook.com/groups/408004796247683/permalink/431283740586455/
* GitHub Gist:
https://gist.github.com/korakot/fe26c65dc9eed467f4497f784a805716
"""
from __future__ import annotations
import re
from collections import defaultdict
from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from collections.abc import Iterator
from pythainlp.util import Trie
from pythainlp.tokenize import word_dict_trie
[docs]
class LatticeString(str): # noqa: SLOT000
"""Keep the possible tokenizations of a string."""
unique: bool
multi: list[str]
in_dict: bool
def __new__( # noqa: PYI034
cls,
value: str,
multi: Optional[list[str]] = None,
in_dict: bool = True,
) -> LatticeString:
"""Create a new lattice string from a value."""
return str.__new__(cls, value)
[docs]
def __init__(
self,
value: str,
multi: Optional[list[str]] = None,
in_dict: bool = True,
) -> None:
"""
Initialize the lattice string.
:param str value: string value
:param Optional[list[str]] multi: possible tokenizations of the value
:param bool in_dict: whether the value is in the dictionary
"""
self.unique: bool = True
if multi:
self.multi: list[str] = list(multi)
if len(self.multi) > 1:
self.unique = False
else:
self.multi = [value]
self.in_dict: bool = in_dict # if in dictionary
_RE_NONTHAI: str = r"""(?x)
[-a-zA-Z]+| # Latin characters
\d+([,\.]\d+)*| # numbers
[ \t]+| # spaces
\r?\n # newlines
"""
_PAT_NONTHAI: re.Pattern[str] = re.compile(_RE_NONTHAI)
def _serialize(
words_at: defaultdict[int, list[str]], p: int, p2: int
) -> Iterator[str]:
"""Yield word paths from position ``p`` to position ``p2``."""
for w in words_at[p]:
p_ = p + len(w)
if p_ == p2:
yield w
elif p_ < p2:
for path in _serialize(words_at, p_, p2):
yield w + "/" + path
def _find_skip_end(text: str, p: int, custom_dict: Trie) -> int:
"""Find the end of an out-of-dictionary word starting at ``p``."""
len_text = len(text)
m = _PAT_NONTHAI.match(text[p:])
if m: # non-Thai token
return p + m.span()[1]
# Thai token, find minimum skip
for i in range(p, len_text):
ww = custom_dict.prefixes(text[i:])
m = _PAT_NONTHAI.match(text[i:])
if ww or m:
return i
return len_text
def _multicut(
text: str, custom_dict: Optional[Trie] = None
) -> Iterator[LatticeString]:
"""Yield a :class:`LatticeString` for each part of the text."""
if not custom_dict:
custom_dict = word_dict_trie()
len_text = len(text)
words_at: defaultdict[int, list[str]] = defaultdict(list)
q = {0}
last_p = 0 # last position for yield
while min(q) < len_text:
p = min(q)
q -= {p}
for w in custom_dict.prefixes(text[p:]):
words_at[p].append(w)
q.add(p + len(w))
len_q = len(q)
if len_q == 1:
q0 = min(q)
yield LatticeString(
text[last_p:q0], list(_serialize(words_at, last_p, q0))
)
last_p = q0
elif len_q == 0: # no dictionary word
i = _find_skip_end(text, p, custom_dict)
w = text[p:i]
words_at[p].append(w)
yield LatticeString(w, in_dict=False)
last_p = i
q.add(i)
[docs]
def mmcut(text: str) -> list[str]:
"""
Tokenize text into words with the minimum-cut selection.
:param str text: text to be tokenized
:return: list of words
:rtype: list[str]
"""
res = []
for w in _multicut(text):
mm = min(w.multi, key=lambda x: x.count("/"))
res.extend(mm.split("/"))
return res
def _combine(ww: list[LatticeString]) -> Iterator[str]:
if ww == []:
yield ""
else:
w = ww[0]
for tail in _combine(ww[1:]):
if w.unique:
yield w + "|" + tail
else:
for m in w.multi:
yield m.replace("/", "|") + "|" + tail
[docs]
def segment(text: str, custom_dict: Optional[Trie] = None) -> list[str]:
"""
Tokenize text into words with dictionary-based maximum matching.
:param str text: text to be tokenized
:param pythainlp.util.Trie custom_dict: dictionary trie
(default: a trie of :func:`pythainlp.corpus.thai_words`)
:return: list of words
:rtype: list[str]
"""
if not text or not isinstance(text, str):
return []
if not custom_dict:
custom_dict = word_dict_trie()
return list(_multicut(text, custom_dict=custom_dict))
[docs]
def find_all_segment(
text: str, custom_dict: Optional[Trie] = None
) -> list[str]:
"""
Get all possible tokenizations of text.
:param str text: text to be tokenized
:param pythainlp.util.Trie custom_dict: dictionary trie
(default: a trie of :func:`pythainlp.corpus.thai_words`)
:return: list of tokenizations, each with words separated by ``|``
:rtype: list[str]
"""
if not text or not isinstance(text, str):
return []
if not custom_dict:
custom_dict = word_dict_trie()
ww = list(_multicut(text, custom_dict=custom_dict))
return list(_combine(ww))