python-ucto 0.6.8__cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl

python_ucto-0.6.8.data/data/sources/ucto_wrapper.pyx

@@ -0,0 +1,184 @@
+#embedsignature=True
+#*****************************
+# Python-ucto
+#   by Maarten van Gompel
+#   Centre for Language Studies
+#   Radboud University Nijmegen
+#
+#   Licensed under GPLv3
+#****************************/
+
+from libcpp.string cimport string
+from libcpp cimport bool
+from libcpp.vector cimport vector
+from cython.operator cimport dereference as deref, preincrement as inc
+from cython import address
+from libc.stdint cimport *
+from libcpp.utility cimport pair
+import os.path
+import sys
+cimport libfolia_classes
+cimport ucto_classes
+
+UCTODATAVERSION = "0.11"
+
+class TokenRole:
+    NOROLE                      = 0
+    NOSPACE                     = 1
+    BEGINOFSENTENCE             = 2
+    ENDOFSENTENCE               = 4
+    NEWPARAGRAPH                = 8
+    BEGINQUOTE                  = 16
+    ENDQUOTE                    = 32
+    TEMPENDOFSENTENCE           = 64
+
+class Token:
+    def __init__(self, text, tokentype, role):
+        self.text = text
+        self.tokentype = tokentype
+        self.role = role
+
+    def __str__(self):
+        return self.text
+
+    def type(self):
+        return self.tokentype
+
+    def isendofsentence(self):
+        return self.role & TokenRole.ENDOFSENTENCE
+
+    def iseos(self): #for lazy people
+        return self.role & TokenRole.ENDOFSENTENCE
+
+    def isbeginofsentence(self):
+        return self.role & TokenRole.BEGINOFSENTENCE
+
+    def isnewparagraph(self):
+        return self.role & TokenRole.NEWPARAGRAPH
+
+    def isbeginofquote(self):
+        return self.role & TokenRole.BEGINQUOTE
+
+    def isendofquote(self):
+        return self.role & TokenRole.ENDQUOTE
+
+    def nospace(self):
+        return self.role & TokenRole.NOSPACE
+
+
+cdef class Tokenizer:
+    cdef ucto_classes.TokenizerClass tok
+
+    def __init__(self, filename, **kwargs):
+        self.tok.init(filename.encode('utf-8'))
+        for arg, value in kwargs.items():
+            if arg == 'lowercase':
+                self.tok.setLowercase(value is True)
+            elif arg == 'uppercase':
+                self.tok.setUppercase(value is True)
+            elif arg == 'sentencedetection':
+                sys.stderr.write("[python-ucto] Argument 'sentencedetection' is deprecated and has no effect, it is always enabled.\n")
+            elif arg == 'paragraphdetection':
+                self.tok.setParagraphDetection(value is True)
+            elif arg == 'quotedetection':
+                self.tok.setQuoteDetection(value is True)
+            elif arg == 'sentenceperlineinput':
+                self.tok.setSentencePerLineInput(value is True)
+            elif arg == 'sentenceperlineoutput':
+                self.tok.setSentencePerLineOutput(value is True)
+            elif arg == 'xmlinput' or arg == 'foliainput':
+                self.tok.setXMLInput(value is True)
+            elif arg == 'xmloutput' or arg == 'foliaoutput':
+                if 'docid' in kwargs:
+                    docid = kwargs['docid']
+                else:
+                    docid = "untitled"
+                self.tok.setXMLOutput(value is True, docid.encode('utf-8'))
+            elif arg == 'debug':
+                self.tok.setDebug(int(value))
+            elif arg == 'docid':
+                pass
+            else:
+                raise ValueError("No such keyword argument: " +  arg)
+
+    def tokenize(self, str inputfile, str outputfile):
+        """Run ucto from inputfile to outputfile (like command line tool)"""
+        self.tok.tokenize(inputfile.encode('utf-8'), outputfile.encode('utf-8'))
+
+
+
+    def process(self, str line):
+        """Feed text to the tokeniser. This needs not be a single line."""
+        self.tok.tokenizeLine(line.encode('utf-8'))
+
+    def sentences(self):
+        cdef vector[string] results = self.tok.getUTF8Sentences()
+        cdef vector[string].iterator it = results.begin()
+        cdef int sentencecount = len(results)
+        while it != results.end():
+            yield str(deref(it), 'utf-8').replace("<utt>",'')
+            inc(it)
+
+    def lowercase(self):
+        return self.tok.getLowercase()
+
+    def uppercase(self):
+        return self.tok.getLowercase()
+
+    def __iter__(self):
+        cdef vector[ucto_classes.Token] v
+        cdef vector[ucto_classes.Token].iterator it
+        while True:
+            v = self.tok.popSentence()
+            if v.empty():
+                break
+            it = v.begin()
+            while it != v.end():
+                tokentext = str(deref(it).texttostring(), 'utf-8')
+                tokentype = str(deref(it).typetostring(), 'utf-8')
+                role = deref(it).role
+                if self.lowercase():
+                    tokentext = tokentext.lower()
+                elif self.uppercase():
+                    tokentext = tokentext.upper()
+                yield Token(tokentext, tokentype, role)
+                inc(it)
+
+
+def localpath():
+    xdg_config_dir = os.environ.get("XDG_CONFIG_HOME", os.path.join(os.environ.get("HOME",""), ".config"))
+    return os.environ.get("UCTODATAPATH", os.path.join(xdg_config_dir,"ucto") )
+
+def installdata(targetdir=None, version=UCTODATAVERSION):
+    if targetdir is None:
+        targetdir = localpath()
+    else:
+        targetdir = os.path.join(targetdir,"ucto")
+    if os.path.exists(targetdir):
+        print(f"Uctodata configuration directory already exists: {targetdir}, refusing to overwrite, please remove it first if you want to install all data anew.", file=sys.stderr)
+    else:
+        tmpdir=os.environ.get("TMPDIR","/tmp")
+        if os.system(f"cd {tmpdir} && mkdir -p {targetdir} && wget -O uctodata.tar.gz https://github.com/LanguageMachines/uctodata/releases/download/v{version}/uctodata-{version}.tar.gz && tar -xzf uctodata.tar.gz && cd uctodata-{version} && mv config/* {targetdir}/ && cd .. && rm -Rf uctodata-{version} && rm -Rf uctodata.tar.gz") != 0:
+            raise Exception("Installation failed")
+        print(f"Installation of uctodata {version} complete", file=sys.stderr)
+        if os.path.isdir("/usr/share/libexttextcat"):
+            if os.system(f"cd {targetdir} && wget -O textcat.cfg https://raw.githubusercontent.com/LanguageMachines/ucto/master/config/textcat.cfg") != 0:
+                raise Exception("Installation of textcat.cfg failed")
+        else:
+            print("Language detection will not be available unless you install libexttextcat and rerun installdata()", file=sys.stderr)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

python_ucto-0.6.8.dist-info/METADATA

@@ -0,0 +1,20 @@
+Metadata-Version: 2.1
+Name: python-ucto
+Version: 0.6.8
+Summary: This is a Python binding to the tokenizer Ucto. Tokenisation is one of the first step in almost any Natural Language Processing task, yet it is not always as trivial a task as it appears to be. This binding makes the power of the ucto tokeniser available to Python. Ucto itself is a regular-expression based, extensible, and advanced tokeniser written in C++ (https://languagemachines.github.io/ucto).
+Home-page: https://github.com/proycon/python-ucto
+Author: Maarten van Gompel
+Author-email: proycon@anaproy.nl
+License: GPLv3
+Keywords: tokenizer tokenization tokeniser tokenisation nlp computational_linguistics ucto
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: POSIX
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Requires: ucto (>=0.34)
+Requires-Dist: Cython
+

python_ucto-0.6.8.dist-info/RECORD

@@ -0,0 +1,18 @@
+ucto.cpython-310-x86_64-linux-gnu.so,sha256=IpUbCOhQZ2QqBJiVDwRemKGiNLep7dnEXSrTxTJnZvc,2317201
+python_ucto-0.6.8.data/data/sources/ucto_wrapper.pyx,sha256=hsussZVPOXyikT9gtnBjR_aE-eWZhdQnvpopiM5XJRU,6392
+python_ucto.libs/libicuuc-1796a535.so.50.2,sha256=ZPT2z0tpkV6chLxylDTz47Zaw3Gt15-izOGUkLni31M,1796193
+python_ucto.libs/libgomp-a34b3233.so.1.0.0,sha256=On6uznIxkRvi-7Gz58tMtcLg-E4MK7c3OUcrWh_uyME,168193
+python_ucto.libs/libicui18n-97d01360.so.50.2,sha256=yosCXXIz7LxTVHdhu51XJEa_DXTOHfo9REusho0I12c,2733217
+python_ucto.libs/libxml2-174e59c1.so.2.9.14,sha256=YLx-sKMzrajESv4FC5XZlTDceuoSgvlLPCUpRjKeq0Q,6791169
+python_ucto.libs/libfolia-8d74cccd.so.21.0.0,sha256=A-VRpMHjqPGVMsu1GXW4XLEhFsO0Q5P82H7i9eY45iM,27319017
+python_ucto.libs/libexttextcat-2-e165f333.0.so.0.0.0,sha256=FDvo3bKVOKgMSv41BCGw5MYTr_tiWs-xjF4FEVYkq60,21617
+python_ucto.libs/libucto-92c8806e.so.6.0.0,sha256=0wogFlgJgOZMJrCuJpKaf4WgFIqoaicHO39b4NHFlN8,6106761
+python_ucto.libs/libicuio-d92a2ee9.so.50.2,sha256=mJCS6k0yQpjOqeKjoHihzEiDLGej2WpYhwCYsSPIhws,64017
+python_ucto.libs/libticcutils-bcffb764.so.9.0.0,sha256=Ca3KZPlaNQhMn5MQ2crR7489t4vHZ4iUDsOvJAstMjY,8811377
+python_ucto.libs/libicudata-cb3ba60c.so.50.2,sha256=mi096d4G6tWwhWUE-5Q8VeNA2WoUS8GRbeOhH-IBpSk,20787817
+python_ucto.libs/liblzma-004595ca.so.5.2.2,sha256=8QVimqSNA2IHkdqYx4HnesWENyP2gf2uXocOhf7tetc,163473
+python_ucto.libs/libbz2-a273e504.so.1.0.6,sha256=Ks7i35uwgx4aPZalUSQBwyfxeXGQ9BGdNInwVf00yb8,70993
+python_ucto-0.6.8.dist-info/WHEEL,sha256=K6ps4aVUsEay-13wCJy8H3VKOTWNlXTmCBW4TF80uCE,151
+python_ucto-0.6.8.dist-info/RECORD,,
+python_ucto-0.6.8.dist-info/METADATA,sha256=L5sNmcy0FhzyB9J5Y15dZBHcQY-xuJH0UVCC0hYXoeU,1136
+python_ucto-0.6.8.dist-info/top_level.txt,sha256=7zjSe25cKXiD3nFVCqHOi-tWjWL3N2nPFaVCsq3V6mw,5

python_ucto-0.6.8.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: setuptools (74.1.2)
+Root-Is-Purelib: false
+Tag: cp310-cp310-manylinux_2_17_x86_64
+Tag: cp310-cp310-manylinux2014_x86_64
+

python_ucto-0.6.8.dist-info/top_level.txt

@@ -0,0 +1 @@
+ucto