fugashi 1.5.2.dev3__cp314-cp314-macosx_10_15_universal2.whl

fugashi/__init__.py

@@ -0,0 +1 @@
+from .fugashi import *

fugashi/cli.py

@@ -0,0 +1,50 @@
+from fugashi import GenericTagger, Tagger, build_dictionary
+import sys
+import fileinput
+
+
+def main():
+    """
+    This is a simple wrapper for fugashi so you can test it from the command line.
+    Like the mecab binary, it treats each line of stdin as one sentence. You can
+    pass tagger arguments here too.
+    """
+    args = " ".join(sys.argv[1:])
+
+    # This should work if you specify a different dictionary,
+    # but it should also work with the pip unidic.
+    # Try the GenericTagger and then try the Unidic tagger.
+    try:
+        tagger = GenericTagger(args, quiet=True)
+    except RuntimeError:
+        tagger = Tagger(args)
+
+    for line in fileinput.input([]):
+        print(tagger.parse(line.strip()))
+
+
+def info():
+    """Print configuration info."""
+    args = " ".join(sys.argv[1:])
+    try:
+        tagger = GenericTagger(args, quiet=True)
+    except RuntimeError:
+        tagger = Tagger(args)
+    # TODO get the fugashi version here too
+    print("Fugashi dictionary info:")
+    print("-----")
+    for di in tagger.dictionary_info:
+        for field in "version size charset filename".split():
+            print((field + ":").ljust(10), di[field])
+        print("-----")
+
+
+def build_dict():
+    """EXPERIMENTAL A wrapper for MeCab's user dictionary building command.
+
+    This also defaults to utf8.
+    """
+    # TODO simplify using pip-installed dictionaries as base
+    args = sys.argv[0] + " -f utf8 -t utf8 " + " ".join(sys.argv[1:])
+    print(args)
+    build_dictionary(args)

fugashi-1.5.2.dev3.dist-info/METADATA

@@ -0,0 +1,165 @@
+Metadata-Version: 2.4
+Name: fugashi
+Version: 1.5.2.dev3
+Summary: Cython MeCab wrapper for fast, pythonic Japanese tokenization.
+Author-email: Paul O'Leary McCann <polm@dampfkraft.com>
+License-Expression: MIT AND BSD-3-Clause
+Project-URL: source, https://github.com/polm/fugashi
+Project-URL: funding, https://github.com/sponsors/polm
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: Natural Language :: Japanese
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+License-File: LICENSE.mecab
+Provides-Extra: unidic
+Requires-Dist: unidic; extra == "unidic"
+Provides-Extra: unidic-lite
+Requires-Dist: unidic-lite; extra == "unidic-lite"
+Dynamic: license-file
+
+[![Open in Streamlit](https://static.streamlit.io/badges/streamlit_badge_black_white.svg)](https://fugashi.streamlit.app)
+[![Current PyPI packages](https://badge.fury.io/py/fugashi.svg)](https://pypi.org/project/fugashi/)
+![Test Status](https://github.com/polm/fugashi/workflows/test-manylinux/badge.svg)
+[![PyPI - Downloads](https://img.shields.io/pypi/dm/fugashi)](https://pypi.org/project/fugashi/)
+![Supported Platforms](https://img.shields.io/badge/platforms-linux%20macosx%20windows-blue)
+
+# fugashi
+
+<img src="https://github.com/polm/fugashi/raw/main/fugashi.png" width=125 height=125 alt="fugashi by Irasutoya" />
+
+fugashi is a Cython wrapper for [MeCab](https://taku910.github.io/mecab/), a
+Japanese tokenizer and morphological analysis tool.  Wheels are provided for
+Linux, OSX (Intel), and Win64, and UniDic is [easy to install](#installing-a-dictionary).
+
+**issueを英語で書く必要はありません。**
+
+Check out the [interactive demo][], see the [blog post](https://www.dampfkraft.com/nlp/fugashi.html) for background
+on why fugashi exists and some of the design decisions, or see [this
+guide][guide] for a basic introduction to Japanese tokenization.
+
+[guide]: https://www.dampfkraft.com/nlp/how-to-tokenize-japanese.html
+[interactive demo]: https://fugashi.streamlit.app
+
+If you are on a platform for which wheels are not provided, you'll need to
+install MeCab first. It's recommended you install [from
+source](https://github.com/taku910/mecab). If you need to build from source on
+Windows, [@chezou's fork](https://github.com/chezou/mecab) is recommended; see
+[issue #44](https://github.com/polm/fugashi/issues/44#issuecomment-954426115)
+for an explanation of the problems with the official repo.
+
+Known platforms without wheels:
+
+- musl-based distros like alpine [#77](https://github.com/polm/fugashi/issues/77)
+- PowerPC
+- Windows 32bit
+
+## Usage
+
+```python
+from fugashi import Tagger
+
+tagger = Tagger('-Owakati')
+text = "麩菓子は、麩を主材料とした日本の菓子。"
+tagger.parse(text)
+# => '麩 菓子 は 、 麩 を 主材 料 と し た 日本 の 菓子 。'
+for word in tagger(text):
+    print(word, word.feature.lemma, word.pos, sep='\t')
+    # "feature" is the Unidic feature data as a named tuple
+```
+
+## Installing a Dictionary
+
+fugashi requires a dictionary. [UniDic](https://unidic.ninjal.ac.jp/) is
+recommended, and two easy-to-install versions are provided.
+
+  - [unidic-lite](https://github.com/polm/unidic-lite), a slightly modified version 2.1.2 of Unidic (from 2013) that's relatively small
+  - [unidic](https://github.com/polm/unidic-py), the latest UniDic 3.1.0, which is 770MB on disk and requires a separate download step
+
+If you just want to make sure things work you can start with `unidic-lite`, but
+for more serious processing `unidic` is recommended. For production use you'll
+generally want to generate your own dictionary too; for details see the [MeCab
+documentation](https://taku910.github.io/mecab/learn.html).
+
+To get either of these dictionaries, you can install them directly using `pip`
+or do the below:
+
+```sh
+pip install 'fugashi[unidic-lite]'
+
+# The full version of UniDic requires a separate download step
+pip install 'fugashi[unidic]'
+python -m unidic download
+```
+
+For more information on the different MeCab dictionaries available, see [this article](https://www.dampfkraft.com/nlp/japanese-tokenizer-dictionaries.html).
+
+## Dictionary Use
+
+fugashi is written with the assumption you'll use Unidic to process Japanese,
+but it supports arbitrary dictionaries. 
+
+If you're using a dictionary besides Unidic you can use the GenericTagger like this:
+
+```python
+from fugashi import GenericTagger
+tagger = GenericTagger()
+
+# parse can be used as normal
+tagger.parse('something')
+# features from the dictionary can be accessed by field numbers
+for word in tagger(text):
+    print(word.surface, word.feature[0])
+```
+
+You can also create a dictionary wrapper to get feature information as a named tuple. 
+
+```python
+from fugashi import GenericTagger, create_feature_wrapper
+CustomFeatures = create_feature_wrapper('CustomFeatures', 'alpha beta gamma')
+tagger = GenericTagger(wrapper=CustomFeatures)
+for word in tagger.parseToNodeList(text):
+    print(word.surface, word.feature.alpha)
+```
+
+## Citation
+
+If you use fugashi in research, it would be appreciated if you cite this paper. You can read it at [the ACL Anthology](https://www.aclweb.org/anthology/2020.nlposs-1.7/) or [on Arxiv](https://arxiv.org/abs/2010.06858).
+
+    @inproceedings{mccann-2020-fugashi,
+        title = "fugashi, a Tool for Tokenizing {J}apanese in Python",
+        author = "McCann, Paul",
+        booktitle = "Proceedings of Second Workshop for NLP Open Source Software (NLP-OSS)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://www.aclweb.org/anthology/2020.nlposs-1.7",
+        pages = "44--51",
+        abstract = "Recent years have seen an increase in the number of large-scale multilingual NLP projects. However, even in such projects, languages with special processing requirements are often excluded. One such language is Japanese. Japanese is written without spaces, tokenization is non-trivial, and while high quality open source tokenizers exist they can be hard to use and lack English documentation. This paper introduces fugashi, a MeCab wrapper for Python, and gives an introduction to tokenizing Japanese.",
+    }
+
+## Alternatives
+
+If you have a problem with fugashi feel free to open an issue. However, there
+are some cases where it might be better to use a different library.
+
+- If you don't want to deal with installing MeCab at all, try [SudachiPy](https://github.com/WorksApplications/sudachi.rs).
+- If you need to work with Korean, try [pymecab-ko](https://github.com/NoUnique/pymecab-ko) or [KoNLPy](https://konlpy.org/en/latest/).
+
+## License and Copyright Notice
+
+fugashi is released under the terms of the [MIT license](./LICENSE). Please
+copy it far and wide.
+
+fugashi is a wrapper for MeCab, and fugashi wheels include MeCab binaries.
+MeCab is copyrighted free software by Taku Kudo `<taku@chasen.org>` and Nippon
+Telegraph and Telephone Corporation, and is redistributed under the [BSD
+License](./LICENSE.mecab).

fugashi-1.5.2.dev3.dist-info/RECORD

@@ -0,0 +1,11 @@
+fugashi-1.5.2.dev3.dist-info/RECORD,,
+fugashi-1.5.2.dev3.dist-info/WHEEL,sha256=MzHA8aA11K4m-mjpjTobbWUqTlhv5RdoytLBFWQb_k4,142
+fugashi-1.5.2.dev3.dist-info/entry_points.txt,sha256=jV282mMQTVkhqOVFTdm_ZQ03pJndByW2JtrSa_a2Wms,121
+fugashi-1.5.2.dev3.dist-info/top_level.txt,sha256=1CQTgPUFi4hjTQg2nHdIR-oH6EfyXtpLhiUglCmuOoM,8
+fugashi-1.5.2.dev3.dist-info/METADATA,sha256=vqmoeA67uuR_uYp9sP88G2NqQdnmzCONPtag9FLCNAo,7328
+fugashi-1.5.2.dev3.dist-info/licenses/LICENSE,sha256=5LkFbx-2Q0xAeg8Y4DVBmBkFBNfz3rvXTG6heUdwXac,1076
+fugashi-1.5.2.dev3.dist-info/licenses/LICENSE.mecab,sha256=YvSyNFCprUDk24Bj1FwdE8eNB8onya2mLEypwRwfPns,1602
+fugashi/fugashi.cpython-314-darwin.so,sha256=vz1rYnjILoTfq2FFTOeHrVWqBbX6LkmqhkWiJ5Jo6c8,399712
+fugashi/__init__.py,sha256=WsYoQIj7b2S1O5aXOsyhh4tBmFGLkkZ5rrtHdohAQic,23
+fugashi/cli.py,sha256=tQXqi_8D0c0s7X1MZHyzWwmj7S9SkfaIeMwn53iJRQQ,1505
+fugashi/.dylibs/libmecab.2.dylib,sha256=SdFs5STxn4cA55Izqw7xHkPLC8ojeeQbE_btbKbbdWE,1009472

fugashi-1.5.2.dev3.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: false
+Tag: cp314-cp314-macosx_10_15_universal2
+Generator: delocate 0.13.0
+

fugashi-1.5.2.dev3.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+fugashi = fugashi.cli:main
+fugashi-build-dict = fugashi.cli:build_dict
+fugashi-info = fugashi.cli:info

fugashi-1.5.2.dev3.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Paul O'Leary McCann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fugashi-1.5.2.dev3.dist-info/licenses/LICENSE.mecab

@@ -0,0 +1,29 @@
+Copyright (c) 2001-2008, Taku Kudo
+Copyright (c) 2004-2008, Nippon Telegraph and Telephone Corporation
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are
+permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above
+   copyright notice, this list of conditions and the
+   following disclaimer.
+
+ * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the
+   following disclaimer in the documentation and/or other
+   materials provided with the distribution.
+
+ * Neither the name of the Nippon Telegraph and Telegraph Corporation
+   nor the names of its contributors may be used to endorse or
+   promote products derived from this software without specific
+   prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

fugashi-1.5.2.dev3.dist-info/top_level.txt

@@ -0,0 +1 @@
+fugashi