ltk-data 1.0.0__py3-none-any.whl

ltk_data/__init__.py

@@ -0,0 +1,26 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import os
+from importlib.metadata import version as metaversion
+
+__version__ = metaversion('ltk_data')
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+
+# pylint:disable=wrong-import-position
+from ltk_data.corpora import *  # isort:skip
+from ltk_data.utils import pickler  # isort:skip
+from ltk_data.utils import load_pickle  # isort:skip
+from ltk_data.path import add_nltk_path  # isort:skip
+
+# nltk requires env setup before first import
+ltk_data = os.path.join(ROOT, 'ltk_data')
+
+add_nltk_path(ltk_data)

ltk_data/compile.py

@@ -0,0 +1,23 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import utilo
+
+import ltk_data
+
+
+def compile_names():
+    utilo.log('compile names')
+    ltk_data.pickler(ltk_data.NAMES_FAMILY)
+    ltk_data.pickler(ltk_data.NAMES_FEMALE)
+    ltk_data.pickler(ltk_data.NAMES_MALE)
+
+
+if __name__ == "__main__":
+    compile_names()

ltk_data/corpora/__init__.py

@@ -0,0 +1,21 @@
+# =============================================================================
+# C O P Y R I G H T
+# -----------------------------------------------------------------------------
+# Copyright (c) 2021-2022 by Helmut Konrad Fahrendholz. All rights reserved.
+# This file is property of Helmut Konrad Fahrendholz. Any unauthorized copy,
+# use or distribution is an offensive act against international law and may
+# be prosecuted under federal law. Its content is company confidential.
+# =============================================================================
+
+import os
+
+import ltk_data
+
+CORPORA = os.path.join(ltk_data.ROOT, 'ltk_data/corpora')
+
+NAMES = os.path.join(CORPORA, 'names')
+NAMES_MALE = os.path.join(NAMES, 'male')
+NAMES_FEMALE = os.path.join(NAMES, 'female')
+NAMES_FAMILY = os.path.join(NAMES, 'family')
+
+STOPWORDS = os.path.join(CORPORA, 'stopwords')

ltk_data/corpora/crubadan/README

@@ -0,0 +1,35 @@
+Language Id Corpus
+Kevin Scannell
+
+This directory contains 3-gram frequencies for 449 writing systems 
+gathered by the web crawler "An Crúbadán", as of 11 April 2010.
+See http://borel.slu.edu/crubadan/ for more information.
+
+The web crawler works at the level of "writing systems" vs. "languages",
+so for example Serbian Cyrillic and Serbian Latin are treated
+separately, as are Portuguese as spoken in Brazil vs. Portugal, etc.
+The 3-gram files are named using 2- or 3-letter "writing system codes"
+that were never intended to be exposed to the outside world.
+We are working on establishing a mapping between our codes and
+the writing systems laid out in Oliver Streiter's XNL-RDF database.  
+
+The file table.txt lists all 449 writing systems.  The first column
+contains the internal Crúbadán code, the second column contains the
+ISO 639-3 code for the language represented by the writing system, and
+the third column is an English language description.
+
+Copyright 2010 Kevin P. Scannell <kscanne at gmail dot com>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+