python-ucto 0.6.10__cp313-cp313-musllinux_1_1_x86_64.whl

python_ucto-0.6.10.data/data/sources/ucto_wrapper.pyx

@@ -0,0 +1,184 @@
+#embedsignature=True
+#*****************************
+# Python-ucto
+#   by Maarten van Gompel
+#   Centre for Language Studies
+#   Radboud University Nijmegen
+#
+#   Licensed under GPLv3
+#****************************/
+
+from libcpp.string cimport string
+from libcpp cimport bool
+from libcpp.vector cimport vector
+from cython.operator cimport dereference as deref, preincrement as inc
+from cython import address
+from libc.stdint cimport *
+from libcpp.utility cimport pair
+import os.path
+import sys
+cimport libfolia_classes
+cimport ucto_classes
+
+UCTODATAVERSION = "0.11"
+
+class TokenRole:
+    NOROLE                      = 0
+    NOSPACE                     = 1
+    BEGINOFSENTENCE             = 2
+    ENDOFSENTENCE               = 4
+    NEWPARAGRAPH                = 8
+    BEGINQUOTE                  = 16
+    ENDQUOTE                    = 32
+    TEMPENDOFSENTENCE           = 64
+
+class Token:
+    def __init__(self, text, tokentype, role):
+        self.text = text
+        self.tokentype = tokentype
+        self.role = role
+
+    def __str__(self):
+        return self.text
+
+    def type(self):
+        return self.tokentype
+
+    def isendofsentence(self):
+        return self.role & TokenRole.ENDOFSENTENCE
+
+    def iseos(self): #for lazy people
+        return self.role & TokenRole.ENDOFSENTENCE
+
+    def isbeginofsentence(self):
+        return self.role & TokenRole.BEGINOFSENTENCE
+
+    def isnewparagraph(self):
+        return self.role & TokenRole.NEWPARAGRAPH
+
+    def isbeginofquote(self):
+        return self.role & TokenRole.BEGINQUOTE
+
+    def isendofquote(self):
+        return self.role & TokenRole.ENDQUOTE
+
+    def nospace(self):
+        return self.role & TokenRole.NOSPACE
+
+
+cdef class Tokenizer:
+    cdef ucto_classes.TokenizerClass tok
+
+    def __init__(self, filename, **kwargs):
+        self.tok.init(filename.encode('utf-8'))
+        for arg, value in kwargs.items():
+            if arg == 'lowercase':
+                self.tok.setLowercase(value is True)
+            elif arg == 'uppercase':
+                self.tok.setUppercase(value is True)
+            elif arg == 'sentencedetection':
+                sys.stderr.write("[python-ucto] Argument 'sentencedetection' is deprecated and has no effect, it is always enabled.\n")
+            elif arg == 'paragraphdetection':
+                self.tok.setParagraphDetection(value is True)
+            elif arg == 'quotedetection':
+                self.tok.setQuoteDetection(value is True)
+            elif arg == 'sentenceperlineinput':
+                self.tok.setSentencePerLineInput(value is True)
+            elif arg == 'sentenceperlineoutput':
+                self.tok.setSentencePerLineOutput(value is True)
+            elif arg == 'xmlinput' or arg == 'foliainput':
+                self.tok.setXMLInput(value is True)
+            elif arg == 'xmloutput' or arg == 'foliaoutput':
+                if 'docid' in kwargs:
+                    docid = kwargs['docid']
+                else:
+                    docid = "untitled"
+                self.tok.setXMLOutput(value is True, docid.encode('utf-8'))
+            elif arg == 'debug':
+                self.tok.setDebug(int(value))
+            elif arg == 'docid':
+                pass
+            else:
+                raise ValueError("No such keyword argument: " +  arg)
+
+    def tokenize(self, str inputfile, str outputfile):
+        """Run ucto from inputfile to outputfile (like command line tool)"""
+        self.tok.tokenize(inputfile.encode('utf-8'), outputfile.encode('utf-8'))
+
+
+
+    def process(self, str line):
+        """Feed text to the tokeniser. This needs not be a single line."""
+        self.tok.tokenizeLine(line.encode('utf-8'))
+
+    def sentences(self):
+        cdef vector[string] results = self.tok.getUTF8Sentences()
+        cdef vector[string].iterator it = results.begin()
+        cdef int sentencecount = len(results)
+        while it != results.end():
+            yield str(deref(it), 'utf-8').replace("<utt>",'')
+            inc(it)
+
+    def lowercase(self):
+        return self.tok.getLowercase()
+
+    def uppercase(self):
+        return self.tok.getLowercase()
+
+    def __iter__(self):
+        cdef vector[ucto_classes.Token] v
+        cdef vector[ucto_classes.Token].iterator it
+        while True:
+            v = self.tok.popSentence()
+            if v.empty():
+                break
+            it = v.begin()
+            while it != v.end():
+                tokentext = str(deref(it).texttostring(), 'utf-8')
+                tokentype = str(deref(it).typetostring(), 'utf-8')
+                role = deref(it).role
+                if self.lowercase():
+                    tokentext = tokentext.lower()
+                elif self.uppercase():
+                    tokentext = tokentext.upper()
+                yield Token(tokentext, tokentype, role)
+                inc(it)
+
+
+def localpath():
+    xdg_config_dir = os.environ.get("XDG_CONFIG_HOME", os.path.join(os.environ.get("HOME",""), ".config"))
+    return os.environ.get("UCTODATAPATH", os.path.join(xdg_config_dir,"ucto") )
+
+def installdata(targetdir=None, version=UCTODATAVERSION):
+    if targetdir is None:
+        targetdir = localpath()
+    else:
+        targetdir = os.path.join(targetdir,"ucto")
+    if os.path.exists(targetdir):
+        print(f"Uctodata configuration directory already exists: {targetdir}, refusing to overwrite, please remove it first if you want to install all data anew.", file=sys.stderr)
+    else:
+        tmpdir=os.environ.get("TMPDIR","/tmp")
+        if os.system(f"cd {tmpdir} && mkdir -p {targetdir} && wget -O uctodata.tar.gz https://github.com/LanguageMachines/uctodata/releases/download/v{version}/uctodata-{version}.tar.gz && tar -xzf uctodata.tar.gz && cd uctodata-{version} && mv config/* {targetdir}/ && cd .. && rm -Rf uctodata-{version} && rm -Rf uctodata.tar.gz") != 0:
+            raise Exception("Installation failed")
+        print(f"Installation of uctodata {version} complete", file=sys.stderr)
+        if os.path.isdir("/usr/share/libexttextcat"):
+            if os.system(f"cd {targetdir} && wget -O textcat.cfg https://raw.githubusercontent.com/LanguageMachines/ucto/master/config/textcat.cfg") != 0:
+                raise Exception("Installation of textcat.cfg failed")
+        else:
+            print("Language detection will not be available unless you install libexttextcat and rerun installdata()", file=sys.stderr)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

python_ucto-0.6.10.dist-info/METADATA

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: python-ucto
+Version: 0.6.10
+Summary: This is a Python binding to the tokenizer Ucto. Tokenisation is one of the first step in almost any Natural Language Processing task, yet it is not always as trivial a task as it appears to be. This binding makes the power of the ucto tokeniser available to Python. Ucto itself is a regular-expression based, extensible, and advanced tokeniser written in C++ (https://languagemachines.github.io/ucto).
+Home-page: https://github.com/proycon/python-ucto
+Author: Maarten van Gompel
+Author-email: proycon@anaproy.nl
+License: GPL-3.0-only
+Keywords: tokenizer tokenization tokeniser tokenisation nlp computational_linguistics ucto
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: POSIX
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Requires: ucto (>=0.36)
+Requires-Dist: Cython
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: requires
+Dynamic: requires-dist
+Dynamic: summary

python_ucto-0.6.10.dist-info/RECORD

@@ -0,0 +1,20 @@
+ucto.cpython-313-x86_64-linux-musl.so,sha256=sU2vnCSzSW0rAa1YIlkVM8fLpFagQDhcL9CM9MD8j9I,2724945
+python_ucto-0.6.10.dist-info/METADATA,sha256=h3BR4GkyhTa1ijcp0ai7dmuzA3w7hU1Si596CyYnBpg,1236
+python_ucto-0.6.10.dist-info/WHEEL,sha256=TIU9MrPwV2TTDER1sJLMhWI27WMvHL-X55XCyK-2nY8,113
+python_ucto-0.6.10.dist-info/top_level.txt,sha256=7zjSe25cKXiD3nFVCqHOi-tWjWL3N2nPFaVCsq3V6mw,5
+python_ucto-0.6.10.dist-info/RECORD,,
+python_ucto-0.6.10.data/data/sources/ucto_wrapper.pyx,sha256=hsussZVPOXyikT9gtnBjR_aE-eWZhdQnvpopiM5XJRU,6392
+python_ucto.libs/liblzma-712ecf44.so.5.2.5,sha256=7kvrZ6mnhB-8_rW2_AEv7H3i3sRDIqDUJaRIzu2lAfo,142889
+python_ucto.libs/libgomp-cd951527.so.1.0.0,sha256=U2zda7o5P6DwCQHNyzt7Hw13pk9wKHfh-c4ZgCzbfsU,187009
+python_ucto.libs/libfolia-cd722aa4.so.22.0.0,sha256=OW0UC9lUraf-mFE7wT3wz0mG2R7erf6bmKSJmKUR29M,36662377
+python_ucto.libs/libicudata-33bdc345.so.67.1,sha256=W7tSBH0i_bTl4D-LWm6PfW2ga60jf0bD4w951ILuNmg,28405913
+python_ucto.libs/libicuio-a3ad3dd8.so.67.1,sha256=P1Xrzbhf6OVvKWhDd8-XUb5xJgJ8wHuADjvzYbpn76E,60353
+python_ucto.libs/libbz2-e55174b0.so.1.0.8,sha256=nLQu8p2AuUOP8XKyFjOxRcuzbTMhNmeEMKDgu4ZqMzI,62697
+python_ucto.libs/libexttextcat-2-aa3651c4.0.so.0.0.0,sha256=7J4Eo2SdMzAWnFwgHlqIUEPQSF2Z9TUbIUxdyA6Ho6Y,21481
+python_ucto.libs/libucto-8ce70bd9.so.6.0.0,sha256=_tt3DaCNeD229Z4PgRh04EYgvv9itl5Ka683YGT1174,8417425
+python_ucto.libs/libicuuc-98673c7b.so.67.1,sha256=QFgJGr38T8HZa5tMm-WCN1Oh_NJ4W7gxmcxtdYBepZc,2059401
+python_ucto.libs/libgcc_s-a04fdf82.so.1,sha256=YxqJNaesQMhDswHEQpXsiLnVvMBBbYO6KYMDZFPWKSM,81257
+python_ucto.libs/libxml2-0d3d92a0.so.2.9.14,sha256=G5uW2wHM6q8ILK1oYTAsEpi2WhHvESMja4F6NfuL7Ns,1306513
+python_ucto.libs/libticcutils-13aac825.so.10.0.0,sha256=PraCNTuv2cMCaW-Bk13DCfXQq7XdP2qs4HJI7OroDTw,10044985
+python_ucto.libs/libstdc++-a9383cce.so.6.0.28,sha256=Wy9UCdwS1rwI9GU5e7qE61S0AkRqqwti1q_adWSs-Rk,2447393
+python_ucto.libs/libicui18n-17bc7491.so.67.1,sha256=l5BBF2b4GUYXvebfa2BWzCtR1JTLzeuzTXP3HeK-xf4,3750521

python_ucto-0.6.10.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: false
+Tag: cp313-cp313-musllinux_1_1_x86_64
+

python_ucto-0.6.10.dist-info/top_level.txt

@@ -0,0 +1 @@
+ucto