eKoNLPy 2.2.1__tar.gz → 2.2.2__tar.gz

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/CHANGELOG.md

@@ -1,5 +1,28 @@
 <!--next-version-placeholder-->
 
+## v2.2.2 (2026-09-17)
+
+### Bug Fixes
+
+- Harden data loading, correct classifier metric bugs, and modernize typing
+  ([#125](https://github.com/entelecheia/eKoNLPy/pull/125),
+  [`a6144e1`](https://github.com/entelecheia/eKoNLPy/commit/a6144e18f09f680874648dd3e5f1f152e34b8833))
+
+- Resolve fugashi-build-dict via PATH fallback and tighten loader consistency
+  ([#125](https://github.com/entelecheia/eKoNLPy/pull/125),
+  [`a6144e1`](https://github.com/entelecheia/eKoNLPy/commit/a6144e18f09f680874648dd3e5f1f152e34b8833))
+
+- Use forward slashes for dictionary compiler paths on Windows
+  ([#125](https://github.com/entelecheia/eKoNLPy/pull/125),
+  [`a6144e1`](https://github.com/entelecheia/eKoNLPy/commit/a6144e18f09f680874648dd3e5f1f152e34b8833))
+
+### Testing
+
+- Expect normalized dictionary compiler paths in build_userdic assertions
+  ([#125](https://github.com/entelecheia/eKoNLPy/pull/125),
+  [`a6144e1`](https://github.com/entelecheia/eKoNLPy/commit/a6144e18f09f680874648dd3e5f1f152e34b8833))
+
+
 ## v2.2.1 (2026-09-16)
 
 ### Bug Fixes

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: eKoNLPy
-Version: 2.2.1
+Version: 2.2.2
 Summary: A Korean natural language processing toolkit for economic analysis
 Project-URL: Homepage, https://ekonlpy.entelecheia.ai
 Project-URL: Repository, https://github.com/entelecheia/eKoNLPy

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eKoNLPy"
-version = "2.2.1"
+version = "2.2.2"
 description = "A Korean natural language processing toolkit for economic analysis"
 authors = [{ name = "Young Joon Lee", email = "yj.lee@chu.ac.kr" }]
 license = "MIT"
@@ -101,6 +101,7 @@ skip = [
     'docs',
     'tests',
     'venv',
+    '.venv',
     '.copier-template',
     '.refs',
 ]

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/__cli__.py

@@ -2,11 +2,13 @@
 
 # Importing the libraries
 
+from typing import Optional
+
 import click
 
 from ._version import __version__
 
-CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
+CONTEXT_SETTINGS = {"help_option_names": ["-h", "--help"]}
 
 
 @click.command(context_settings=CONTEXT_SETTINGS)
@@ -18,16 +20,16 @@ CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
     default="ekonlpy",
     help="The tagger to use. [ekonlpy|mecab]]",
 )
-@click.option("--input", "-i", help="The input text to tag.")
+@click.option("--input", "-i", "text", help="The input text to tag.")
 @click.pass_context
-def main(ctk, tagger, input):
+def main(ctk: click.Context, tagger: str, text: Optional[str]) -> None:
     """This is the command line interface for eKoNLPy.
 
     It is used to tag Korean text with a Korean morphological analyzer.
     """
     # Print a message to the user.
-    if input:
-        click.echo(tag(tagger, input))
+    if text:
+        click.echo(tag(tagger, text))
     else:
         # Print usage message to the user.
         click.echo(ctk.get_help())

ekonlpy-2.2.2/src/ekonlpy/_version.py

@@ -0,0 +1 @@
+__version__ = "2.2.2"

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/base/base.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Optional, Tuple
+from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -10,13 +10,13 @@ class BaseMecab:
     def parse(
         self,
         text: str,
-    ) -> List[Tuple[str, str]]:
+    ) -> list[tuple[str, str]]:
         raise NotImplementedError
 
     def pos(
         self,
         text: str,
-    ) -> List[Tuple[str, str]]:
+    ) -> list[tuple[str, str]]:
         return self.parse(text)
 
     def tokenize(
@@ -24,7 +24,7 @@ class BaseMecab:
         text: str,
         strip_pos: bool = False,
         postag_delim: str = "/",
-    ) -> List[str]:
+    ) -> list[str]:
         tokens = self.parse(text)
 
         return [
@@ -32,15 +32,15 @@ class BaseMecab:
             for token_pos in tokens
         ]
 
-    def morphs(self, text: str) -> List[str]:
+    def morphs(self, text: str) -> list[str]:
         return self.tokenize(text, strip_pos=True)
 
     def nouns(
         self,
         text: str,
         flatten: bool = True,
-        noun_pos: Optional[List[str]] = None,
-    ) -> List[str]:
+        noun_pos: Optional[list[str]] = None,
+    ) -> list[str]:
         if not noun_pos:
             noun_pos = []
         return [surface for surface, pos in self.pos(text) if pos in noun_pos]

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/data/__init__.py

@@ -1 +1,3 @@
 from .tagset import mecab_tags
+
+__all__ = ["mecab_tags"]

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/data/tagset.py

@@ -124,7 +124,6 @@ skip_chk_tags = {
     ("MM", "SC", "NNG", "SC", "NR"): "NNG",
     ("NNG", "SC", "NNG"): "NNG",
     ("NNG", "SN", "NNG"): "NNG",
-    ("NNG", "SN", "NNG"): "NNG",
     ("NNG", "SN", "NNBC"): "NNG",
     ("NNG", "SN", "NNBC", "NNG"): "NNG",
     ("NNG", "SY", "NNBC", "JX"): "NNG",

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/etag/_template.py

@@ -1,4 +1,4 @@
-from typing import Dict, Set
+from typing import Union
 
 from ekonlpy.data.tagset import nouns_tags, pass_tags, skip_chk_tags, skip_tags
 from ekonlpy.utils.dictionary import TermDictionary
@@ -7,10 +7,10 @@ from ekonlpy.utils.dictionary import TermDictionary
 class ExtTagger:
     dictionary: TermDictionary
     max_ngram: int
-    skip_chk_tags: Dict[str, str]
-    skip_tags: Set[str]
-    nouns_tags: Set[str]
-    pass_tags: Set[str]
+    skip_chk_tags: dict[tuple[str, ...], str]
+    skip_tags: set[str]
+    nouns_tags: set[str]
+    pass_tags: set[tuple[str, str]]
 
     def __init__(
         self,
@@ -24,27 +24,27 @@ class ExtTagger:
         self.nouns_tags = set(nouns_tags)
         self.pass_tags = set(pass_tags)
 
-    def add_skip_chk_tags(self, template):
+    def add_skip_chk_tags(self, template: dict[tuple[str, ...], str]) -> None:
         if isinstance(template, dict):
             self.skip_chk_tags.update(template)
 
-    def add_skip_tags(self, tags):
+    def add_skip_tags(self, tags: Union[list[str], set[str]]) -> None:
         if isinstance(tags, (list, set)):
             self.skip_tags.update(tags)
 
-    def pos(self, tokens):
-        def ctagger(
-            ctokens,
-            max_ngram,
-            cnouns_tags,
-            cpass_tags,
-            cskip_chk_tags,
-            cskip_tags,
-            cdictionary,
-        ):
+    def pos(self, tokens: list[tuple[str, str]]) -> list[tuple[str, str]]:  # noqa: C901
+        def ctagger(  # noqa: C901
+            ctokens: list[tuple[str, str]],
+            max_ngram: int,
+            cnouns_tags: set[str],
+            cpass_tags: set[tuple[str, str]],
+            cskip_chk_tags: dict[tuple[str, ...], str],
+            cskip_tags: set[str],
+            cdictionary: TermDictionary,
+        ) -> list[tuple[str, str]]:
             tokens_org = ctokens
             num_tokens = len(ctokens)
-            tokens_new = []
+            tokens_new: list[tuple[str, str]] = []
             ipos = 0
 
             while ipos < num_tokens:
@@ -53,17 +53,19 @@ class ExtTagger:
                     # if found a word from the dictionary, skip for loop
                     if word_found or ipos + ngram > num_tokens:
                         continue
-                    if any(word.isspace() for word, _ in tokens_org[ipos:ipos + ngram]):
+                    if any(
+                        word.isspace() for word, _ in tokens_org[ipos : ipos + ngram]
+                    ):
                         continue
 
-                    tmp_tags = []
-                    for j in range(ngram):
-                        tmp_tags.append(
+                    tmp_tags = tuple(
+                        (
                             "NNG"
                             if tokens_org[ipos + j][1] in cnouns_tags
                             else tokens_org[ipos + j][1]
                         )
-                    tmp_tags = tuple(tmp_tags)
+                        for j in range(ngram)
+                    )
 
                     if tmp_tags not in cpass_tags:
                         new_word = ""
@@ -75,7 +77,7 @@ class ExtTagger:
                             ipos += ngram
                             word_found = True
 
-                    if not word_found and tmp_tags in cskip_chk_tags.keys():
+                    if not word_found and tmp_tags in cskip_chk_tags:
                         new_word = ""
                         num_word = ""
                         for j in range(ngram):
@@ -107,7 +109,11 @@ class ExtTagger:
             return tokens_new
 
         tokens = [
-            (w, t) if w.isspace() else (w.strip(), self.dictionary.check_tag(w.strip(), t))
+            (
+                (w, t)
+                if w.isspace()
+                else (w.strip(), self.dictionary.check_tag(w.strip(), t))
+            )
             for w, t in tokens
         ]
 
@@ -130,6 +136,9 @@ class ExtTagger:
             self.dictionary,
         )
 
-        tokens = [(w, t if w.isspace() else self.dictionary.check_tag(w, t)) for w, t in tokens]
+        tokens = [
+            (w, t if w.isspace() else self.dictionary.check_tag(w, t))
+            for w, t in tokens
+        ]
 
         return tokens

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/mecab/_mecab.py

@@ -1,7 +1,8 @@
 import logging
 import os
 from collections import namedtuple
-from typing import Any, List, Optional, Sequence, Tuple, Union
+from collections.abc import Sequence
+from typing import Optional, Union
 
 import fugashi as _mecab
 
@@ -29,14 +30,14 @@ Feature = namedtuple(
 #           expression='하/XSA/*+ᄇ니다/EF/*')),
 
 
-def _extract_feature(values: Sequence[Any]) -> Feature:
+def _extract_feature(values: Sequence[object]) -> Feature:
     # Reference:
     # - http://taku910.github.io/mecab/learn.html
     # - https://docs.google.com/spreadsheets/d/1-9blXKjtjeKZqsf4NzHeYJCrr49-nXeRF6D80udfcwY
     # - https://bitbucket.org/eunjeon/mecab-ko-dic/src/master/utils/dictionary/lexicon.py
 
     # feature = <pos>,<semantic>,<has_jongseong>,<reading>,<type>,<start_pos>,<end_pos>,<expression>
-    assert len(values) == 8
+    assert len(values) == 8  # noqa: S101
 
     feature_values = [value if value != "*" else None for value in values]
     feature = dict(zip(Feature._fields, feature_values))
@@ -54,14 +55,14 @@ class Mecab(BaseMecab):
     backend = "fugashi"
     verbose: bool = False
 
-    _tagger: _mecab.GenericTagger  # type: ignore
+    _tagger: _mecab.GenericTagger  # type: ignore[no-any-unimported]
 
     def __init__(
         self,
         dicdir: Optional[str] = None,
         userdic_path: Optional[str] = None,
         verbose: bool = False,
-        **kwargs,
+        **kwargs: object,
     ):
         import mecab_ko_dic
 
@@ -85,22 +86,21 @@ class Mecab(BaseMecab):
                 userdic_path,
             )
         try:
-            self._tagger = _mecab.GenericTagger(MECAB_ARGS)  # type: ignore
+            self._tagger = _mecab.GenericTagger(MECAB_ARGS)
             if self.verbose:
                 dictionary_info = self._tagger.dictionary_info
                 sysdic_path = dictionary_info[0]["filename"]
                 logger.debug("Mecab is loaded from %s", sysdic_path)
         except RuntimeError as e:
-            raise MeCabError(
-                'The MeCab dictionary does not exist at "%s". Is the dictionary correctly installed?\nYou can also try entering the dictionary path when initializing the MeCab class: "MeCab(\'/some/dic/path\')"'
-                % dicdir
+            raise MeCabError(  # noqa: TRY003
+                f'The MeCab dictionary does not exist at "{dicdir}". Is the dictionary correctly installed?\nYou can also try entering the dictionary path when initializing the MeCab class: "MeCab(\'/some/dic/path\')"'
             ) from e
         except NameError as e:
-            raise MeCabError(
+            raise MeCabError(  # noqa: TRY003
                 "The fugashi package is not installed. Please install fugashi with: pip install fugashi"
             ) from e
 
-    def _parse(self, text: str) -> List[Tuple[str, Feature]]:
+    def _parse(self, text: str) -> list[tuple[str, Feature]]:
         return [
             (node.surface, _extract_feature(node.feature))
             for node in self._tagger(text)
@@ -111,7 +111,7 @@ class Mecab(BaseMecab):
         text: str,
         flatten: bool = True,
         include_whitespace_token: bool = False,
-    ) -> List[Tuple[str, str]]:
+    ) -> list[tuple[str, str]]:
         """Tag text, optionally preserving original whitespace runs as SP tokens."""
         res = []
         cursor = 0
@@ -138,19 +138,19 @@ class Mecab(BaseMecab):
         text: str,
         flatten: bool = True,
         include_whitespace_token: bool = False,
-    ) -> List[Tuple[str, str]]:
+    ) -> list[tuple[str, str]]:
         return self.parse(
             text, flatten=flatten, include_whitespace_token=include_whitespace_token
         )
 
-    def tokenize(
+    def tokenize(  # type: ignore[override]
         self,
         text: str,
         flatten: bool = True,
         include_whitespace_token: bool = False,
         strip_pos: bool = False,
         postag_delim: str = "/",
-    ) -> List[str]:
+    ) -> list[str]:
         tokens = self.parse(
             text, flatten=flatten, include_whitespace_token=include_whitespace_token
         )
@@ -160,17 +160,17 @@ class Mecab(BaseMecab):
             for token_pos in tokens
         ]
 
-    def morphs(self, text: str, flatten: bool = True) -> List[str]:
+    def morphs(self, text: str, flatten: bool = True) -> list[str]:
         return self.tokenize(
             text, flatten=flatten, strip_pos=True, include_whitespace_token=False
         )
 
     def nouns(
         self,
-        text: Union[str, List[Tuple[str, str]]],
+        text: Union[str, list[tuple[str, str]]],
         flatten: bool = True,
-        noun_pos: Optional[List[str]] = None,
-    ) -> List[str]:
+        noun_pos: Optional[list[str]] = None,
+    ) -> list[str]:
         if not noun_pos:
             noun_pos = ["NNG", "NNP", "XSN", "SL", "XR", "NNB", "NR"]
         tagged = (

{ekonlpy-2.2.1 → ekonlpy-2.2.2}/src/ekonlpy/mecab/_userdic.py

@@ -1,9 +1,12 @@
 import logging
 import os
+import shutil
 import subprocess
+import sys
 from collections import namedtuple
+from collections.abc import Iterator
 from pathlib import Path
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from typing import Optional
 
 import mecab_ko_dic
 import pandas as pd
@@ -46,23 +49,26 @@ ContextEntry = namedtuple(
 )
 
 
-def iternamedtuples(df):
-    Row = namedtuple("Row", df.columns)
+def iternamedtuples(  # type: ignore[no-any-unimported]
+    df: pd.DataFrame,
+) -> Iterator[DicEntry]:
     for row in df.itertuples():
-        yield Row(*row[1:])
+        yield DicEntry(*row[1:])
 
 
-def has_jongseong(c):
+def has_jongseong(c: str) -> bool:
     return int((ord(c[-1]) - 0xAC00) % 28) != 0
 
 
 class MecabDicConfig:
-    userdic: Dict[str, DicEntry] = {}
+    userdic: dict[str, DicEntry]
     dicdir: str = mecab_ko_dic.DICDIR
-    left_ids: List[ContextEntry] = []
-    right_ids: List[ContextEntry] = []
+    left_ids: list[ContextEntry]
+    right_ids: list[ContextEntry]
+    userdic_path: Optional[str] = None
 
     def __init__(self, userdic_path: Optional[str] = None):
+        self.userdic_path = userdic_path
         if userdic_path:
             self.load_userdic(userdic_path)
         else:
@@ -71,13 +77,13 @@ class MecabDicConfig:
         self.left_ids = self.load_context_ids("left-id.def")
         self.right_ids = self.load_context_ids("right-id.def")
 
-    def load_context_ids(self, id_file: str) -> List[ContextEntry]:
+    def load_context_ids(self, id_file: str) -> list[ContextEntry]:
         id_file = os.path.join(self.dicdir, id_file)
         context_ids = []
-        with open(id_file, "r", encoding="utf-8") as f:
+        with open(id_file, encoding="utf-8") as f:
             for line in f:
-                id, vals = line.split()
-                entry = ContextEntry(id, *vals.split(","))
+                entry_id, vals = line.split()
+                entry = ContextEntry(entry_id, *vals.split(","))
                 context_ids.append(entry)
         return context_ids
 
@@ -85,6 +91,7 @@ class MecabDicConfig:
         for entry in self.left_ids:
             if entry.pos == search.pos and entry.semantic == search.semantic:
                 return entry.id
+        return None
 
     def find_right_context_id(self, search: DicEntry) -> Optional[str]:
         for entry in self.right_ids:
@@ -94,19 +101,20 @@ class MecabDicConfig:
                 and entry.has_jongseong == search.has_jongseong
             ):
                 return entry.id
+        return None
 
-    def load_userdic(self, userdic_path: str):
+    def load_userdic(self, userdic_path: str) -> None:
         userdic_path_ = Path(userdic_path)
 
         if userdic_path_.is_dir():
             self.userdic = {}
             for f in userdic_path_.glob("*.csv"):
                 df = pd.read_csv(f, names=DicEntry._fields)
-                dic = {e.surface: DicEntry(*e) for e in iternamedtuples(df)}
+                dic = {e.surface: e for e in iternamedtuples(df)}
                 self.userdic = {**self.userdic, **dic}
         else:
             df = pd.read_csv(userdic_path_, names=DicEntry._fields)
-            self.userdic = {e.surface: DicEntry(*e) for e in iternamedtuples(df)}
+            self.userdic = {e.surface: e for e in iternamedtuples(df)}
         logger.info("No. of user dictionary entires loaded: %d", len(self.userdic))
 
     def add_entry_to_userdic(
@@ -116,7 +124,7 @@ class MecabDicConfig:
         semantic: str = "*",
         reading: Optional[str] = None,
         cost: int = 1000,
-    ):
+    ) -> None:
         entry = DicEntry(
             surface=surface,
             cost=cost,
@@ -131,7 +139,7 @@ class MecabDicConfig:
         )
         self.userdic[surface] = entry
 
-    def adjust_context_ids(self):
+    def adjust_context_ids(self) -> None:
         for entry in self.userdic.values():
             entry = entry._replace(
                 left_id=self.find_left_context_id(entry),
@@ -139,11 +147,11 @@ class MecabDicConfig:
             )
             self.userdic[entry.surface] = entry
 
-    def adjust_costs(self, cost: int = 1000):
+    def adjust_costs(self, cost: int = 1000) -> None:
         for surface, entry in self.userdic.items():
             self.userdic[surface] = entry._replace(cost=cost)
 
-    def save_userdic(self, save_path: str):
+    def save_userdic(self, save_path: str) -> None:
         if len(self.userdic) > 0:
             df = pd.DataFrame(self.userdic.values())
             df.to_csv(save_path, header=False, index=False)
@@ -153,11 +161,35 @@ class MecabDicConfig:
         else:
             logger.warning("No user dictionary entries to save.")
 
+    @staticmethod
+    def _build_dict_executable() -> str:
+        executable = shutil.which("fugashi-build-dict")
+        if executable:
+            return executable
+        binary = "fugashi-build-dict.exe" if os.name == "nt" else "fugashi-build-dict"
+        return os.path.join(os.path.dirname(sys.executable), binary)
+
+    @staticmethod
+    def _build_dict_path(path: str) -> str:
+        # The MeCab dictionary compiler eats backslashes as escape characters
+        # in its arguments, so Windows paths must use forward slashes.
+        return path.replace("\\", "/") if os.name == "nt" else path
+
     def build_userdic(
         self, built_userdic_path: str, userdic_path: Optional[str] = None
-    ):
+    ) -> None:
         if userdic_path:
             self.userdic_path = userdic_path
-        args = f'-d "{self.dicdir}" -u "{built_userdic_path}" {self.userdic_path}'
-        # print(args)
-        subprocess.run(["fugashi-build-dict", args])
+        if not self.userdic_path:
+            raise ValueError(  # noqa: TRY003
+                "userdic_path is not set; call save_userdic() first or pass userdic_path."
+            )
+        cmd = [
+            self._build_dict_executable(),
+            "-d",
+            self._build_dict_path(self.dicdir),
+            "-u",
+            self._build_dict_path(built_userdic_path),
+            self._build_dict_path(self.userdic_path),
+        ]
+        subprocess.run(cmd, check=True)  # noqa: S603