lcpcli 0.2.9__tar.gz → 0.3.1__tar.gz

{lcpcli-0.2.9 → lcpcli-0.3.1}/.gitignore

@@ -2,6 +2,7 @@ __pycache__/
 lcpcli/__pycache__/
 tmp.py
 *.pyc
+.env
 build/
 dist/
 dist/

{lcpcli-0.2.9 → lcpcli-0.3.1}/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright 2025 LiRI - UZH
+Copyright 2026 LiRI - UZH
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

lcpcli-0.3.1/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: lcpcli
+Version: 0.3.1
+Summary: Helper for converting CONLLU files and uploading the corpus to LiRI Corpus Platform (LCP)
+Project-URL: Homepage, https://github.com/liri-uzh/lcpcli
+Project-URL: Issues, https://github.com/liri-uzh/lcpcli/issues
+Author-email: Danny McDonald <daniel.mcdonald@uzh.ch>, Igor Mustač <igor.mustac@uzh.ch>, Jeremy Zehr <jeremy.zehr@uzh.ch>, Jonathan Schaber <jeremy.schaber@uzh.ch>
+License: MIT
+License-File: LICENSE.txt
+Keywords: CONLL,TEI,VERT,corpora,corpus,linguistics
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Requires-Dist: diskcache>=5.6.3
+Requires-Dist: jsonpickle>=3.0
+Requires-Dist: jsonschema>=4.21
+Requires-Dist: lxml>=4.7.1
+Requires-Dist: pandas>=2.2.2
+Requires-Dist: py7zr>=0.20.5
+Requires-Dist: requests>=2.30.0
+Requires-Dist: tqdm>=4.65.0
+Requires-Dist: tuspy==1.1.0
+Requires-Dist: types-requests>=2.30.0.0
+Requires-Dist: types-tqdm>=4.65.0.1
+Requires-Dist: xmltodict>=0.13
+Description-Content-Type: text/markdown
+
+# LCP CLI module
+
+> Command-line tool for converting CONLLU files and uploading the corpus to LCP
+
+## Installation
+
+Make sure you have python 3.11+ with `pip` installed in your local environment, then run:
+
+```bash
+pip install lcpcli
+```
+
+## Usage
+
+**Examples:**
+
+Conversion of a CoNLL-U (Plus) corpus:
+
+```bash
+lcpcli -i ~/conll_ext/ -o ~/upload/
+```
+
+Data upload:
+
+```bash
+lcpcli -c ~/upload/ -k $API_KEY -s $API_SECRET -p "my project" --live
+```
+
+Including `--live` points the upload to the live instance of LCP. Leave it out if you want to add a corpus to an instance of LCP running on `localhost`.
+
+**Help:**
+
+```bash
+lcpcli --help
+```
+
+`lcpcli` can take a corpus of CoNLL-U (PLUS) files and import it to a collection created on LCP.
+
+Besides the standard token-level CoNLL-U fields (`form`, `lemma`, `upos`, `xpos`, `feats`, `head`, `deprel`, `deps`) one can also provide document-, paragraph- and sentence-level annotations using comment lines in the files (see [the CoNLL-U Format section](#conll-u-format)).
+
+### CoNLL-U Format
+
+The CoNLL-U format is documented at: https://universaldependencies.org/format.html
+
+The LCP CLI converter will treat all the comments that start with `# newdoc KEY = VALUE` as document-level attributes, and all the comments that start with `# newpar KEY = VALUE` as paragraph-level attributes. All other comment lines following the format `# key = value` will be treated sentence-level attributes.
+
+The key-value pairs in the `FEATS` and `MISC` columns of a token line will be mapped to corresponding attributes in the LCP corpus. Additionally, if the `MISC` cell includes `SpaceAfter=Yes` or `SpaceAfter=No` (case senstive) the token will be represented with (respectively, without) a trailing space character in the database.
+
+#### CoNLL-U Plus
+
+CoNLL-U Plus is an extension to the CoNLLU-U format documented at: https://universaldependencies.org/ext-format.html
+
+If your files start with a comment line of the form `# global.columns = ID FORM LEMMA UPOS XPOS FEATS HEAD DEPREL DEPS MISC`, `lcpcli` will treat them as CoNLL-U PLUS files and process the columns according to the names you set in that line.
+
+### CoNLL-U conversion and upload
+
+1. Create a directory in which you have all your properly-fromatted CoNLL-U files.
+
+2. Visit an LCP instance (e.g. _catchphrase_) and create a new collection if you don't already have one where your corpus should go.
+
+3. Retrieve the API key and secret for your project by clicking on the button that says: "Create API Key".
+
+4. Once you have your API key and secret, you can start converting and uploading your corpus by running the following command:
+
+```
+lcpcli -i $CONLLU_FOLDER -o $OUTPUT_FOLDER -k $API_KEY -s $API_SECRET -p $PROJECT_NAME --live
+```
+
+- `$CONLLU_FOLDER` should point to the folder that contains your CONLLU files
+- `$OUTPUT_FOLDER` should point to *another* folder that will be used to store the converted files to be uploaded
+- `$API_KEY` is the key you copied from your project on LCP (still visible when you visit the page)
+- `$API_SECRET` is the secret you copied from your project on LCP (only visible upon API Key creation)
+- `$PROJECT_NAME` is the name of the project exactly as displayed on LCP -- it is case-sensitive, and space characters should be escaped
+
+### Other input formats, rich data
+
+Previous versions of `lcpcli` defined procedures to include rich annotations in CoNLL-U files, including time-anchored media files, in combination with annex non-CoNLL-U files. These methods are no longer supported -- use an older version of `lcpcli` if you require those features.
+
+`lcpcli` now ships with a Python module called `lcpcli.builder` that you can use to convert any input format. The default CoNLL-U converter included in `lcpcli` uses `lcpcli.builder` under the hood.
+
+You can find a short tutorial on how to use the module [in BUILDER.md](BUILDER.md). Further information can be found in [the LCP documentation](https://lcp.linguistik.uzh.ch/manual/builder.html).

lcpcli-0.3.1/README.md

@@ -0,0 +1,81 @@
+# LCP CLI module
+
+> Command-line tool for converting CONLLU files and uploading the corpus to LCP
+
+## Installation
+
+Make sure you have python 3.11+ with `pip` installed in your local environment, then run:
+
+```bash
+pip install lcpcli
+```
+
+## Usage
+
+**Examples:**
+
+Conversion of a CoNLL-U (Plus) corpus:
+
+```bash
+lcpcli -i ~/conll_ext/ -o ~/upload/
+```
+
+Data upload:
+
+```bash
+lcpcli -c ~/upload/ -k $API_KEY -s $API_SECRET -p "my project" --live
+```
+
+Including `--live` points the upload to the live instance of LCP. Leave it out if you want to add a corpus to an instance of LCP running on `localhost`.
+
+**Help:**
+
+```bash
+lcpcli --help
+```
+
+`lcpcli` can take a corpus of CoNLL-U (PLUS) files and import it to a collection created on LCP.
+
+Besides the standard token-level CoNLL-U fields (`form`, `lemma`, `upos`, `xpos`, `feats`, `head`, `deprel`, `deps`) one can also provide document-, paragraph- and sentence-level annotations using comment lines in the files (see [the CoNLL-U Format section](#conll-u-format)).
+
+### CoNLL-U Format
+
+The CoNLL-U format is documented at: https://universaldependencies.org/format.html
+
+The LCP CLI converter will treat all the comments that start with `# newdoc KEY = VALUE` as document-level attributes, and all the comments that start with `# newpar KEY = VALUE` as paragraph-level attributes. All other comment lines following the format `# key = value` will be treated sentence-level attributes.
+
+The key-value pairs in the `FEATS` and `MISC` columns of a token line will be mapped to corresponding attributes in the LCP corpus. Additionally, if the `MISC` cell includes `SpaceAfter=Yes` or `SpaceAfter=No` (case senstive) the token will be represented with (respectively, without) a trailing space character in the database.
+
+#### CoNLL-U Plus
+
+CoNLL-U Plus is an extension to the CoNLLU-U format documented at: https://universaldependencies.org/ext-format.html
+
+If your files start with a comment line of the form `# global.columns = ID FORM LEMMA UPOS XPOS FEATS HEAD DEPREL DEPS MISC`, `lcpcli` will treat them as CoNLL-U PLUS files and process the columns according to the names you set in that line.
+
+### CoNLL-U conversion and upload
+
+1. Create a directory in which you have all your properly-fromatted CoNLL-U files.
+
+2. Visit an LCP instance (e.g. _catchphrase_) and create a new collection if you don't already have one where your corpus should go.
+
+3. Retrieve the API key and secret for your project by clicking on the button that says: "Create API Key".
+
+4. Once you have your API key and secret, you can start converting and uploading your corpus by running the following command:
+
+```
+lcpcli -i $CONLLU_FOLDER -o $OUTPUT_FOLDER -k $API_KEY -s $API_SECRET -p $PROJECT_NAME --live
+```
+
+- `$CONLLU_FOLDER` should point to the folder that contains your CONLLU files
+- `$OUTPUT_FOLDER` should point to *another* folder that will be used to store the converted files to be uploaded
+- `$API_KEY` is the key you copied from your project on LCP (still visible when you visit the page)
+- `$API_SECRET` is the secret you copied from your project on LCP (only visible upon API Key creation)
+- `$PROJECT_NAME` is the name of the project exactly as displayed on LCP -- it is case-sensitive, and space characters should be escaped
+
+### Other input formats, rich data
+
+Previous versions of `lcpcli` defined procedures to include rich annotations in CoNLL-U files, including time-anchored media files, in combination with annex non-CoNLL-U files. These methods are no longer supported -- use an older version of `lcpcli` if you require those features.
+
+`lcpcli` now ships with a Python module called `lcpcli.builder` that you can use to convert any input format. The default CoNLL-U converter included in `lcpcli` uses `lcpcli.builder` under the hood.
+
+You can find a short tutorial on how to use the module [in BUILDER.md](BUILDER.md). Further information can be found in [the LCP documentation](https://lcp.linguistik.uzh.ch/manual/builder.html).

{lcpcli-0.2.9 → lcpcli-0.3.1}/lcpcli/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.2.9"
+__version__ = "0.3.1"
 
 from .lcpcli import Lcpcli  # noqa: F401

{lcpcli-0.2.9 → lcpcli-0.3.1}/lcpcli/builder.py

@@ -4,8 +4,8 @@ import csv
 import json
 import os
 import re
-import tempfile
 import shutil
+import tempfile
 
 from typing import Any
 from uuid import uuid4
@@ -16,9 +16,9 @@ ANCHORINGS = ("stream", "time", "location")
 # ATYPES = ("text", "categorical", "number", "dict", "labels")
 ATYPES_LOOKUP = ("text", "dict", "labels")
 NAMEDATALEN = 63
-PATTERN_TXT = (
-    "(must start with a lower case and only contain alpha-numerical characters)"
-)
+PATTERN_TXT = "(must start with a lower case, be at leat 2 characters long and only contain alpha-numerical characters)"
+
+IS_NUM = re.compile(r"^[0-9]+(\.[0-9]+)?$")
 
 
 def meta_subattr(meta: dict, k: str, v: Any) -> dict:
@@ -28,11 +28,7 @@ def meta_subattr(meta: dict, k: str, v: Any) -> dict:
     sub_attr = meta.setdefault(k, {})
     if isinstance(v, list):
         sub_attr["type"] = "labels"
-    elif (
-        isinstance(v, (int, float))
-        or isinstance(v, str)
-        and v.replace(".", "", 1).isdigit()
-    ):
+    elif isinstance(v, (int, float)) or isinstance(v, str) and IS_NUM.match(v):
         sub_attr["type"] = "text" if sub_attr.get("type") == "text" else "number"
     elif isinstance(v, dict):
         sub_attr["type"] = "dict"
@@ -50,14 +46,17 @@ def get_layer_method(layer: "Layer"):
             corpus._layers.pop(layer._name, "")
             fname = f"{layer._name.lower()}.csv"
             corpus._files[fname].close()
+            os.unlink(corpus._files[fname].name)
             corpus._files.pop(fname)
             return GlobalAttribute(corpus, layer._name, args[0])
         largs = [a for a in args]
-        if layer._name == corpus._token and isinstance(largs[0], str):
+        if layer._name == corpus._token and largs and isinstance(largs[0], str):
             form = largs.pop(0)
             layer.form = form
         if len(largs) > 0:
-            assert all(isinstance(c, Layer) for c in largs), RuntimeError()
+            assert all(isinstance(c, Layer) for c in largs), RuntimeError(
+                "Tried to pass non-layers as arguments of a layer"
+            )
             layer.add(*largs)
         for aname, avalue in kwargs.items():
             setattr(layer, aname, avalue)
@@ -85,10 +84,23 @@ def get_layer_method(layer: "Layer"):
                 a.make()
             return
         # source is the attribute that's missing in at least one layer
-        source_a = next(
-            ra for ra in relation_attrs if any(ra not in a._attributes for a in args)
-        )
-        target_a = next(ra for ra in relation_attrs if ra != source_a)
+        try:
+            source_a = next(
+                ra
+                for ra in relation_attrs
+                if any(ra not in a._attributes for a in args)
+            )
+        except:
+            # if can't identify a missing attribute,
+            # look for "head"/"source" or use the first one
+            source_a = next(
+                (ra for ra in relation_attrs if ra in ("head", "source")),
+                next(ra for ra in relation_attrs),
+            )
+        target_a = next((ra for ra in relation_attrs if ra != source_a), None)
+        if not target_a:
+            # no target: possibly a one-token sentence, return
+            return
         # reference nested sets by target's id
         nested_sets = {
             a._attributes[target_a]._value._id: NestedSet(
@@ -246,7 +258,7 @@ class Corpus:
                     can_categorize = (
                         not (is_token and aname in ("form", "lemma"))
                         and len(lookup) <= 100
-                        and all(len(v) < NAMEDATALEN for v in lookup)
+                        and all(len(str(v)) < NAMEDATALEN for v in lookup)
                     )
                     if can_categorize:
                         texts_to_categorical[na] = aname
@@ -326,7 +338,7 @@ class Corpus:
                 "corpusDescription": self._corpus_description,
                 "date": self._date,
                 "url": self._url,
-                "revision": 1,
+                "revision": self._revision,
             },
             "firstClass": {
                 "token": self._token,
@@ -374,7 +386,11 @@ class Corpus:
                 if ais_global:
                     aopts["isGlobal"] = True
                 if aopts["type"] == "categorical" and not ais_global:
-                    aopts["values"] = [v for v in mapping.lookups[aname] if v]
+                    aopts["values"] = [
+                        str(v)
+                        for v in mapping.lookups[aname]
+                        if v is not None and v != ""
+                    ]
                 elif aopts["type"] == "ref":
                     aopts.pop("type")
                     aopts.pop("nullable", "")
@@ -418,14 +434,23 @@ class Layer:
             assert re.match(r"[a-z][a-zA-Z0-9_]+$", name), RuntimeError(
                 f"The attribute '{name}' on the layer {self._name} does not match the pattern {PATTERN_TXT}"
             )
+            # Disallow linebreak in token string values because it messes with CSV's (in particular, FTS)
+            if (
+                self._name == self._corpus._token
+                and isinstance(value, str)
+                and ("\n" in value or "\r" in value)
+            ):
+                print(
+                    f"Warning: a token attribute contains a linebreak; this is not allowed, removing the linebreaks from the value {value}."
+                )
+                value = value.replace("\n", "").replace("\r", "")
             Attribute(self, name, value)
 
     def __getattribute__(self, name: str):
         if re.match(r"[A-Z]", name):
             corpus = self._corpus
             layer = corpus._add_layer(name)
-            self._contains.append(layer)
-            layer._parents.append(self)
+            self.add(layer)
             return get_layer_method(layer)
         return super().__getattribute__(name)
 
@@ -471,7 +496,47 @@ class Layer:
             ch.append(c)
         return ch
 
-    def make(self):
+    def _update_parents_anchors(self):
+        """Update the anchors of all the parents (recursively)"""
+        if not self._made:
+            return
+        corpus = self._corpus
+        parents = self._parents
+        while parents:
+            current_parents = [*parents]
+            parents = []
+            for parent in current_parents:
+                parents += parent._parents
+                for anc_name, anchors in self._anchorings.items():
+                    if anc_name not in parent._anchorings:
+                        parent._anchorings[anc_name] = [*anchors]
+                    parent_anchors = parent._anchorings[anc_name]
+                    if anchors[0] < parent_anchors[0]:
+                        parent_anchors[0] = anchors[0]
+                    if anc_name == "time" and parent._name == corpus._document:
+                        if corpus._upperFrameDocument < parent_anchors[0]:
+                            parent_anchors[0] = corpus._upperFrameDocument
+                        corpus._upperFrameDocument = parent_anchors[1]
+                    if anc_name != "location":
+                        if anchors[1] > parent_anchors[1]:
+                            parent_anchors[1] = anchors[1]
+                        continue
+                    if anchors[1] < parent_anchors[1]:
+                        parent_anchors[1] = anchors[1]
+                    if anchors[2] > parent_anchors[2]:
+                        parent_anchors[2] = anchors[2]
+                    if anchors[3] > parent_anchors[3]:
+                        parent_anchors[3] = anchors[3]
+
+    def clear(self):
+        if not self._made:
+            return
+        # Prepare for deletion: no pointers to other layers/global attributes
+        self._parents = []
+        self._contains = []
+        self._attributes = {}
+
+    def make(self, clear=False):
         if self._made:
             return
         corpus = self._corpus
@@ -494,6 +559,9 @@ class Layer:
             rows.append(doc_name)
             rows.append(json.dumps(self._media))
         if is_token:
+            assert "form" in self._attributes, RuntimeError(
+                "Tried to make a token with no form"
+            )
             seg_parent = self._find_in_parents(corpus._segment)
             rows.append(seg_parent._id)
             char_low = corpus._char_counter
@@ -502,35 +570,8 @@ class Layer:
             )
             self._anchorings["stream"] = [char_low, corpus._char_counter]
         elif self._contains:
-            unset_anchorings = {a for a in ANCHORINGS if not self._anchorings.get(a)}
             for child in self._contains:
                 child.make()
-                if child._name not in mapping.contains:
-                    mapping.contains.append(child._name)
-                # Anchorings
-                for a in unset_anchorings:
-                    if a not in child._anchorings:
-                        continue
-                    child_a = child._anchorings[a]
-                    if a not in self._anchorings:
-                        self._anchorings[a] = [*child_a]
-                    self_a = self._anchorings[a]
-                    if child_a[0] < self_a[0]:
-                        self_a[0] = child_a[0]
-                    if a == "time" and self._name == corpus._document:
-                        if corpus._upperFrameDocument < self_a[0]:
-                            self_a[0] = corpus._upperFrameDocument
-                        corpus._upperFrameDocument = self_a[1]
-                    if a != "location":
-                        if child_a[1] > self_a[1]:
-                            self_a[1] = child_a[1]
-                        continue
-                    if child_a[1] < self_a[1]:
-                        self_a[1] = child_a[1]
-                    if child_a[2] > self_a[2]:
-                        self_a[2] = child_a[2]
-                    if child_a[3] > self_a[3]:
-                        self_a[3] = child_a[3]
             if is_segment:
                 tokens = [
                     ch._attributes.values()
@@ -568,15 +609,27 @@ class Layer:
             rows.append(v)
         # Add any new attribute to mapping
         for aname, attr in self._attributes.items():
+            atype = attr._type
             if aname in mapping.attributes:
+                mattr = mapping.attributes[aname]
+                if atype == "text" and mattr["type"] != atype:
+                    mattr["type"] = "text"
+                else:
+                    try:
+                        mapping.attributes[aname]["subtype"] = attr._subtype
+                    except:
+                        pass
                 continue
-            atype = attr._type
             mapping.attributes[aname] = {
                 "type": atype,
                 "nullable": (
                     True if mapping.counter > 1 else False
                 ),  # adding a new attribute
             }
+            try:
+                mapping.attributes[aname]["subtype"] = attr._subtype
+            except:
+                pass
             if atype == "ref":
                 mapping.attributes[aname]["ref"] = attr._ref.lower()
             elif atype in ATYPES_LOOKUP and aname != "meta":
@@ -656,6 +709,9 @@ class Layer:
             rows.append("" if val == None else str(val))
         mapping.csvs["_main"].writerow(rows)
         self._made = True
+        self._update_parents_anchors()
+        if clear:
+            self.clear()
         return self
 
     def set_time(self, *args):
@@ -711,19 +767,25 @@ class Layer:
         return self
 
     def add(self, *layers: "Layer"):
-        assert not self._contains or all(
-            l._name == self._contains[0]._name for l in layers
-        ), RuntimeError("All the children of a layer must be of the same type")
+        # assert not self._contains or all(
+        #     l._name == self._contains[0]._name for l in layers
+        # ), RuntimeError("All the children of a layer must be of the same type")
         self._contains += layers
+        mapping = self._corpus._layers[self._name]
         for layer in layers:
+            if layer._name not in mapping.contains:
+                mapping.contains.append(layer._name)
             if self not in layer._parents:
                 layer._parents.append(self)
+            layer._update_parents_anchors()
         return self
 
 
 class Attribute:
     def __init__(self, layer: Layer, name: str, value: Any = None):
         self._name = name
+        if name not in layer._attributes:
+            layer._attributes[name] = self
         self._value = value
         self._layer = layer
         self._ref = None
@@ -739,6 +801,10 @@ class Attribute:
             self._value = json.dumps(sorted_dict(value))
         elif isinstance(value, (int, float)):
             atype = "number"
+            if isinstance(value, float):
+                self._subtype = "float"
+                # overwrite to ensure subtype is taken into consideration
+                layer._attributes[name] = self
         elif isinstance(value, GlobalAttribute):
             atype = "ref"
             self._ref = value._name
@@ -747,7 +813,6 @@ class Attribute:
             atype = "entity"
             self._ref = value._name
         self._type: str = atype
-        layer._attributes[name] = self
 
 
 class GlobalAttribute: