jpreprocess 0.1.6__cp313-cp313-win_amd64.whl

jpreprocess/__init__.py

@@ -0,0 +1,33 @@
+import os
+from .jpreprocess import (
+    __version__,
+    JPreprocess,
+    build_dictionary,
+    JPREPROCESS_VERSION,
+)
+from .dictionary import download_dictionary, dictionary_path
+
+
+__all__ = [
+    "JPreprocess",
+    "build_dictionary",
+    "download_dictionary",
+    "JPREPROCESS_VERSION",
+]
+
+
+def jpreprocess(dictionary_version: str = f"v{JPREPROCESS_VERSION}", user_dictionary: str | None = None) -> JPreprocess:
+    """
+    Create jpreprocess instance with naist-jdic dictionary.
+
+    If the system dictionary is not present, this function will download it.
+
+    Arguments:
+    - dictionary_version (str): Version of dictionary to download.
+      We don't recommend specifying this argument unless you are aware of what you are doing.
+    - user_dictionary (str | None): Path to user dictionary. The extionsion must be ".csv" or ".bin".
+    """
+    dict_path = dictionary_path(dictionary_version)
+    if not os.path.exists(dict_path):
+        download_dictionary(dictionary_version)
+    return JPreprocess(dict_path, user_dictionary)

jpreprocess/dictionary.py

@@ -0,0 +1,51 @@
+from contextlib import ExitStack
+import sys
+import atexit
+
+# from https://github.com/r9y9/pyopenjtalk/pull/74
+
+if sys.version_info >= (3, 9):
+    from importlib.resources import as_file, files
+else:
+    from importlib_resources import as_file, files
+
+_file_manager = ExitStack()
+atexit.register(_file_manager.close)
+_file_ref = files(__package__)
+
+
+def dictionary_path(version: str) -> str:
+    return str(_file_manager.enter_context(
+        as_file(_file_ref / version / "naist-jdic")))
+
+
+def download_dictionary(version: str) -> str:
+    from urllib.request import urlopen
+    import tarfile
+    import tempfile
+
+    if version == "latest":
+        url = f"https://github.com/jpreprocess/jpreprocess/releases/latest/download/naist-jdic-jpreprocess.tar.gz"
+    else:
+        url = f"https://github.com/jpreprocess/jpreprocess/releases/download/{version}/naist-jdic-jpreprocess.tar.gz"
+
+    target_dir = _file_ref / version
+
+    with tempfile.TemporaryFile() as file:
+        print('Downloading: "{}"'.format(url))
+        with urlopen(url) as response:
+            try:
+                from tqdm.auto import tqdm
+                with tqdm.wrapattr(file, "write", total=getattr(response, "length", None)) as tar:
+                    for chunk in response:
+                        tar.write(chunk)
+            except ImportError:
+                for chunk in response:
+                    file.write(chunk)
+        file.seek(0)
+        print("Extracting tar file")
+        with tarfile.open(mode="r|gz", fileobj=file) as f, as_file(target_dir) as dir:
+            f.extractall(path=dir)
+        print("done")
+
+    return str(target_dir)

jpreprocess/jpreprocess.pyi

@@ -0,0 +1,94 @@
+from typing import TypedDict, Literal, overload
+
+__version__: str
+JPREPROCESS_VERSION: str
+
+
+class NjdObject(TypedDict):
+    string: str
+    pos: str
+    pos_group1: str
+    pos_group2: str
+    pos_group3: str
+    ctype: str
+    cform: str
+    orig: str
+    read: str
+    pron: str
+    acc: int
+    mora_size: int
+    chain_rule: str
+    chain_flag: int
+
+
+class JPreprocess:
+    def __init__(self, dictionary: str, user_dictionary: str |
+                 None = None) -> None: ...
+
+    def run_frontend(self, text: str) -> list[NjdObject]:
+        """
+        Run text-processing frontend.
+
+        Arguments:
+        - text (str): Input japanese text.
+
+        Returns:
+        - list[NjdObject]: list of NJDNode(s).
+        """
+    def make_label(self, njd_features: list[NjdObject]) -> list[str]:
+        """
+        Make full-context label using NjdObject
+
+        Arguments:
+        - njd_features (list[NjdObject]): list of NJDNode(s).
+
+        Returns:
+        - list[str]: list of full-context labels.
+        """
+    def extract_fullcontext(self, text: str) -> list[str]:
+        """
+        Extract full-context label from the input text.
+
+        Arguments:
+        - text (str): Input japanese text.
+
+        Returns:
+        - list[str]: list of full-context labels.
+        """
+
+    @overload
+    def g2p(self, text: str, kana: bool = False,
+            join: Literal[True] = True) -> str: ...
+
+    @overload
+    def g2p(self, text: str, kana: bool = False, *,
+            join: Literal[False]) -> list[str]: ...
+
+    @overload
+    def g2p(self, text: str, kana: bool,
+            join: Literal[False]) -> list[str]:
+        """
+        Grapheme-to-phoneme (G2P) conversion.
+
+        Arguments:
+        - text (str): Input japanese text.
+        - kana (bool): Whether to generate alphabetical phoneme (False) or kana (True).
+        - join (bool): Whether to generate a list of phoneme or kana (False) or join the output by delimiter (True).
+
+        Returns:
+        - list[str] (when join = False): list of phoneme (when kana = False) or kana (when kana = True).
+        - str (when join = True): list of phoneme joined with space (when kana = False) or kana joined with empty string (when kana = True).
+        """
+
+
+def build_dictionary(input: str, output: str, user: bool = False,
+                     serializer: Literal["jpreprocess", "lindera"] = "jpreprocess") -> None:
+    """
+    Build dictionary binary file(s).
+
+    Arguments:
+    - input (str): Path to source directory (system dictionary) or file (user dictionary).
+    - output (str): Path to destination directory (system dictionary) or file (user dictionary).
+    - user (bool): Whether to build system dictionary (False) or user dictionary (True). Default to False.
+    - serializer (str): The name of serializer to use ("lindera" or "jpreprocess"). Default to "jpreprocess"
+    """

jpreprocess-0.1.6.dist-info/METADATA

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: jpreprocess
+Version: 0.1.6
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development
+Classifier: Intended Audience :: Science/Research
+Classifier: Intended Audience :: Developers
+Requires-Dist: importlib-resources ; python_full_version < '3.9'
+Requires-Dist: pytest>=8.0.1 ; extra == 'dev'
+Requires-Dist: tqdm ; extra == 'progress'
+Provides-Extra: dev
+Provides-Extra: progress
+Requires-Python: >=3.8, <3.15

jpreprocess-0.1.6.dist-info/RECORD

@@ -0,0 +1,9 @@
+jpreprocess/__init__.py,sha256=9M2JDa43Z4iddAPp9hvL-zNNlM21mm3gPIe_VpRuPik,1039
+jpreprocess/dictionary.py,sha256=NcyypwcL3yX-h6-KmiAqd00E-0D5Hc2snlXJJ8NegRM,1637
+jpreprocess/jpreprocess.cp313-win_amd64.pyd,sha256=1B7QVGTmdywHDvKeG44CXFZ7XGa2F3_f6AHQm5S5NP4,3188224
+jpreprocess/jpreprocess.pyi,sha256=Lx6vCk-x7x3B7cjhLZH6-oBdYnyni6o00i-NLxoMLOE,2867
+jpreprocess/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+jpreprocess-0.1.6.dist-info/METADATA,sha256=yiUCuGx0zxZDlrLSKQUV6iJ0DrrYVV1ULqFDiRcSSLA,696
+jpreprocess-0.1.6.dist-info/WHEEL,sha256=z3sDn4xNPtieBDo9mUKkT1e80gbhCuRsQQi1_g6mdQM,97
+jpreprocess-0.1.6.dist-info/sboms/jpreprocess-python.cyclonedx.json,sha256=br7Ex7ZUtRjHlJWJ2d3XF-FPuA15P0zAl3N1owMqNTQ,203162
+jpreprocess-0.1.6.dist-info/RECORD,,

jpreprocess-0.1.6.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.14.1)
+Root-Is-Purelib: false
+Tag: cp313-cp313-win_amd64