lcpcli 0.3.0__tar.gz → 0.3.1__tar.gz

{lcpcli-0.3.0 → lcpcli-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lcpcli
-Version: 0.3.0
+Version: 0.3.1
 Summary: Helper for converting CONLLU files and uploading the corpus to LiRI Corpus Platform (LCP)
 Project-URL: Homepage, https://github.com/liri-uzh/lcpcli
 Project-URL: Issues, https://github.com/liri-uzh/lcpcli/issues
@@ -22,6 +22,7 @@ Requires-Dist: pandas>=2.2.2
 Requires-Dist: py7zr>=0.20.5
 Requires-Dist: requests>=2.30.0
 Requires-Dist: tqdm>=4.65.0
+Requires-Dist: tuspy==1.1.0
 Requires-Dist: types-requests>=2.30.0.0
 Requires-Dist: types-tqdm>=4.65.0.1
 Requires-Dist: xmltodict>=0.13

{lcpcli-0.3.0 → lcpcli-0.3.1}/lcpcli/__init__.py

@@ -1,3 +1,3 @@
-__version__ = "0.3.0"
+__version__ = "0.3.1"
 
 from .lcpcli import Lcpcli  # noqa: F401

{lcpcli-0.3.0 → lcpcli-0.3.1}/lcpcli/builder.py

@@ -18,6 +18,8 @@ ATYPES_LOOKUP = ("text", "dict", "labels")
 NAMEDATALEN = 63
 PATTERN_TXT = "(must start with a lower case, be at leat 2 characters long and only contain alpha-numerical characters)"
 
+IS_NUM = re.compile(r"^[0-9]+(\.[0-9]+)?$")
+
 
 def meta_subattr(meta: dict, k: str, v: Any) -> dict:
     """
@@ -26,11 +28,7 @@ def meta_subattr(meta: dict, k: str, v: Any) -> dict:
     sub_attr = meta.setdefault(k, {})
     if isinstance(v, list):
         sub_attr["type"] = "labels"
-    elif (
-        isinstance(v, (int, float))
-        or isinstance(v, str)
-        and v.replace(".", "", 1).isdigit()
-    ):
+    elif isinstance(v, (int, float)) or isinstance(v, str) and IS_NUM.match(v):
         sub_attr["type"] = "text" if sub_attr.get("type") == "text" else "number"
     elif isinstance(v, dict):
         sub_attr["type"] = "dict"
@@ -48,6 +46,7 @@ def get_layer_method(layer: "Layer"):
             corpus._layers.pop(layer._name, "")
             fname = f"{layer._name.lower()}.csv"
             corpus._files[fname].close()
+            os.unlink(corpus._files[fname].name)
             corpus._files.pop(fname)
             return GlobalAttribute(corpus, layer._name, args[0])
         largs = [a for a in args]
@@ -339,7 +338,7 @@ class Corpus:
                 "corpusDescription": self._corpus_description,
                 "date": self._date,
                 "url": self._url,
-                "revision": 1,
+                "revision": self._revision,
             },
             "firstClass": {
                 "token": self._token,
@@ -387,7 +386,11 @@ class Corpus:
                 if ais_global:
                     aopts["isGlobal"] = True
                 if aopts["type"] == "categorical" and not ais_global:
-                    aopts["values"] = [str(v) for v in mapping.lookups[aname] if v]
+                    aopts["values"] = [
+                        str(v)
+                        for v in mapping.lookups[aname]
+                        if v is not None and v != ""
+                    ]
                 elif aopts["type"] == "ref":
                     aopts.pop("type")
                     aopts.pop("nullable", "")
@@ -447,8 +450,7 @@ class Layer:
         if re.match(r"[A-Z]", name):
             corpus = self._corpus
             layer = corpus._add_layer(name)
-            self._contains.append(layer)
-            layer._parents.append(self)
+            self.add(layer)
             return get_layer_method(layer)
         return super().__getattribute__(name)
 
@@ -494,7 +496,47 @@ class Layer:
             ch.append(c)
         return ch
 
-    def make(self):
+    def _update_parents_anchors(self):
+        """Update the anchors of all the parents (recursively)"""
+        if not self._made:
+            return
+        corpus = self._corpus
+        parents = self._parents
+        while parents:
+            current_parents = [*parents]
+            parents = []
+            for parent in current_parents:
+                parents += parent._parents
+                for anc_name, anchors in self._anchorings.items():
+                    if anc_name not in parent._anchorings:
+                        parent._anchorings[anc_name] = [*anchors]
+                    parent_anchors = parent._anchorings[anc_name]
+                    if anchors[0] < parent_anchors[0]:
+                        parent_anchors[0] = anchors[0]
+                    if anc_name == "time" and parent._name == corpus._document:
+                        if corpus._upperFrameDocument < parent_anchors[0]:
+                            parent_anchors[0] = corpus._upperFrameDocument
+                        corpus._upperFrameDocument = parent_anchors[1]
+                    if anc_name != "location":
+                        if anchors[1] > parent_anchors[1]:
+                            parent_anchors[1] = anchors[1]
+                        continue
+                    if anchors[1] < parent_anchors[1]:
+                        parent_anchors[1] = anchors[1]
+                    if anchors[2] > parent_anchors[2]:
+                        parent_anchors[2] = anchors[2]
+                    if anchors[3] > parent_anchors[3]:
+                        parent_anchors[3] = anchors[3]
+
+    def clear(self):
+        if not self._made:
+            return
+        # Prepare for deletion: no pointers to other layers/global attributes
+        self._parents = []
+        self._contains = []
+        self._attributes = {}
+
+    def make(self, clear=False):
         if self._made:
             return
         corpus = self._corpus
@@ -528,35 +570,8 @@ class Layer:
             )
             self._anchorings["stream"] = [char_low, corpus._char_counter]
         elif self._contains:
-            unset_anchorings = {a for a in ANCHORINGS if not self._anchorings.get(a)}
             for child in self._contains:
                 child.make()
-                if child._name not in mapping.contains:
-                    mapping.contains.append(child._name)
-                # Anchorings
-                for a in unset_anchorings:
-                    if a not in child._anchorings:
-                        continue
-                    child_a = child._anchorings[a]
-                    if a not in self._anchorings:
-                        self._anchorings[a] = [*child_a]
-                    self_a = self._anchorings[a]
-                    if child_a[0] < self_a[0]:
-                        self_a[0] = child_a[0]
-                    if a == "time" and self._name == corpus._document:
-                        if corpus._upperFrameDocument < self_a[0]:
-                            self_a[0] = corpus._upperFrameDocument
-                        corpus._upperFrameDocument = self_a[1]
-                    if a != "location":
-                        if child_a[1] > self_a[1]:
-                            self_a[1] = child_a[1]
-                        continue
-                    if child_a[1] < self_a[1]:
-                        self_a[1] = child_a[1]
-                    if child_a[2] > self_a[2]:
-                        self_a[2] = child_a[2]
-                    if child_a[3] > self_a[3]:
-                        self_a[3] = child_a[3]
             if is_segment:
                 tokens = [
                     ch._attributes.values()
@@ -594,13 +609,17 @@ class Layer:
             rows.append(v)
         # Add any new attribute to mapping
         for aname, attr in self._attributes.items():
+            atype = attr._type
             if aname in mapping.attributes:
-                try:
-                    mapping.attributes[aname]["subtype"] = attr._subtype
-                except:
-                    pass
+                mattr = mapping.attributes[aname]
+                if atype == "text" and mattr["type"] != atype:
+                    mattr["type"] = "text"
+                else:
+                    try:
+                        mapping.attributes[aname]["subtype"] = attr._subtype
+                    except:
+                        pass
                 continue
-            atype = attr._type
             mapping.attributes[aname] = {
                 "type": atype,
                 "nullable": (
@@ -690,6 +709,9 @@ class Layer:
             rows.append("" if val == None else str(val))
         mapping.csvs["_main"].writerow(rows)
         self._made = True
+        self._update_parents_anchors()
+        if clear:
+            self.clear()
         return self
 
     def set_time(self, *args):
@@ -745,13 +767,17 @@ class Layer:
         return self
 
     def add(self, *layers: "Layer"):
-        assert not self._contains or all(
-            l._name == self._contains[0]._name for l in layers
-        ), RuntimeError("All the children of a layer must be of the same type")
+        # assert not self._contains or all(
+        #     l._name == self._contains[0]._name for l in layers
+        # ), RuntimeError("All the children of a layer must be of the same type")
         self._contains += layers
+        mapping = self._corpus._layers[self._name]
         for layer in layers:
+            if layer._name not in mapping.contains:
+                mapping.contains.append(layer._name)
             if self not in layer._parents:
                 layer._parents.append(self)
+            layer._update_parents_anchors()
         return self
 
 

{lcpcli-0.3.0 → lcpcli-0.3.1}/lcpcli/check_files.py

@@ -1,10 +1,10 @@
 import csv
 import json
 import os
+import re
 import sys
 
 from jsonschema import validate
-from re import match, findall
 from typing import Callable
 from uuid import UUID
 
@@ -12,6 +12,8 @@ EXTENSIONS = (".csv", ".tsv")
 LOOKUP_TYPES = ("dict", "text")
 NAMEDATALEN = 63
 
+IS_NUM = re.compile(r"^[0-9]+(\.[0-9]+)?$")
+
 
 def is_lookup(p: dict) -> bool:
     return p.get("type", "") in LOOKUP_TYPES or "ref" in p
@@ -86,16 +88,16 @@ class Checker:
         assert UUID(uuid, version=4), SyntaxError(f"Invalid UUID ({uuid})")
 
     def check_number(self, value: str, aname: str = "") -> None:
-        is_digit = value.strip().isdigit()
-        is_almost_digit = not is_digit and value.strip().replace(".", "", 1).isdigit()
-        if is_almost_digit:
+        is_num = IS_NUM.match(value.strip())
+        is_float = is_num and "." in value
+        if is_float:
             raise TypeError(
                 f"Number attribute {aname} appears to contain floating values ({value}); add 'subtype': 'float' to its properties."
             )
-        assert is_digit, TypeError(f"Number attribute {aname} is ill-formed: {value}")
+        assert is_num, TypeError(f"Number attribute {aname} is ill-formed: {value}")
 
     def check_float(self, value: str, aname: str = "") -> None:
-        is_numerical = value.strip().replace(".", "", 1).isdigit()
+        is_numerical = IS_NUM.match(value.strip())
         assert is_numerical, TypeError(
             f"Float number attribute {aname} is ill-formed: {value}"
         )
@@ -109,6 +111,7 @@ class Checker:
         )
 
     def check_dict(self, str_obj: str, subtyps: dict) -> None:
+        error = ""
         try:
             json_obj = json.loads(str_obj)
             assert isinstance(json_obj, dict), TypeError(
@@ -122,7 +125,7 @@ class Checker:
                     assert (
                         isinstance(v, (int, float))
                         or isinstance(v, str)
-                        and v.replace(".", "", 1).isdigit()
+                        and IS_NUM.match(v)
                     ), TypeError(f"Sub-attribute {k} is not a number ({v})")
                 elif typ in ("labels", "array"):
                     assert isinstance(v, list), TypeError(
@@ -132,15 +135,16 @@ class Checker:
                     assert isinstance(v, str), TypeError(
                         f"Sub-attribute {k} is not a valid text value ({v})"
                     )
-        except:
+        except Exception as e:
             json_obj = None
+            error = str(e)
         assert isinstance(json_obj, dict), SyntaxError(
-            f"Invalid syntax for dict entry ({str_obj})"
+            f"Invalid syntax for dict entry ({str_obj})\n{error}"
         )
         return None
 
     def check_labels(self, bits: str, nbit: int) -> None:
-        assert match(r"^[01]*$", bits), ValueError(
+        assert re.match(r"^[01]*$", bits), ValueError(
             f"Labels column should be series of 0s and 1s, got '{bits}'"
         )
         assert len(bits) == nbit, ValueError(
@@ -151,25 +155,25 @@ class Checker:
     def check_ftsvector(self, vector: str) -> None:
         whole_pattern = r"^('\d+([^']|'')*':\d+(,\d+)*(\s|$))+$"
         simple_unit_pattern = r"('([^']|'')*':[^\s]+)(\s|$)"
-        units = findall(simple_unit_pattern, vector)
+        units = re.findall(simple_unit_pattern, vector)
         for n, (unit, *_) in enumerate(units):
             assert unit.startswith("'"), SyntaxError(
                 f"Each value in the tsvector must start with a single quote character ({unit} -- {n})"
             )
-            assert match(r"'\d+", unit), SyntaxError(
+            assert re.match(r"'\d+", unit), SyntaxError(
                 f"Each value in the tsvector must start with a single quote character followed by an integer index ({unit} -- {n})"
             )
-            m = match(r"'\d+(.*)':\d+(,\d+)*\s?$", unit)
+            m = re.match(r"'\d+(.*)':\d+(,\d+)*\s?$", unit)
             assert m, SyntaxError(
                 f"Each value in the tsvector must end with a single quote followed by a colon and an integer index ({unit} -- {n})"
             )
-        assert match(whole_pattern, vector), SyntaxError(
+        assert re.match(whole_pattern, vector), SyntaxError(
             f"Invalid tsvector string ({vector})"
         )
         return None
 
     def check_range(self, range: str, name: str) -> None:
-        m = match(r"\[(\d+),(\d+)\)", range)
+        m = re.match(r"\[(\d+),(\d+)\)", range)
         assert m, SyntaxError(f"Range '{name}' not in the right format: {range}")
         l, u = (m[1], m[2])
         try:
@@ -192,7 +196,7 @@ class Checker:
         return None
 
     def check_xy_box(self, xy_box: str, name: str) -> None:
-        m = match(r"\((\d+),(\d+)\),\((\d+),(\d+)\)", xy_box)
+        m = re.match(r"\((\d+),(\d+)\),\((\d+),(\d+)\)", xy_box)
         assert m, SyntaxError(f"Range '{name}' not in the right format: {xy_box}")
         x1, y1, x2, y2 = (m[1], m[2], m[3], m[4])
         try:

{lcpcli-0.3.0 → lcpcli-0.3.1}/lcpcli/conllu_builder.py

@@ -19,6 +19,8 @@ CONLLU_COLUMNS = (
     "MISC",
 )
 
+IS_NUM = re.compile(r"^[0-9]+(\.[0-9]+)?$")
+
 
 class LayerProxy:
     dummy_corpus = Corpus("dummy")
@@ -36,10 +38,8 @@ class LayerProxy:
         val = value
         if isinstance(value, str):
             val = value.strip()
-            if val.isdigit():
-                val = int(val)
-            elif val.replace(".", "", 1).isdigit():
-                val = float(val)
+            if IS_NUM.match(val):
+                val = float(val) if "." in val else int(val)
         self.attributes[name.strip()] = val
 
 
@@ -214,10 +214,8 @@ def create_corpus(
                 kwargs: dict = {}
                 for n, x in enumerate(column_names[2:]):
                     kwargs[x] = rest[n]
-                    if rest[n].isdigit():
-                        kwargs[x] = int(rest[n])
-                    elif rest[n].replace(".", "", 1).isdigit():
-                        kwargs[x] = float(rest[n])
+                    if IS_NUM.match(rest[n]):
+                        kwargs[x] = float(rest[n]) if "." in rest[n] else int(rest[n])
                 if "feats" in column_names:
                     feats_idx = column_names.index("feats")
                     feats_str = rest[feats_idx - 2]
@@ -234,7 +232,8 @@ def create_corpus(
                             0 if kwargs["misc"].pop("SpaceAfter") == "No" else 1
                         )
                 token_proxy = TokenProxy()
-                if "head" in kwargs:
+                head_val = kwargs.get("head", "")
+                if head_val and head_val != "_":
                     token_proxy.head = str(int(kwargs.pop("head")))
                 if "deprel" in kwargs:
                     token_proxy.deprel = kwargs.pop("deprel")
@@ -251,11 +250,9 @@ def create_corpus(
             current_doc.entity.make()
         print("Corpus created")
     except Exception as e:
-        raise RuntimeError(
-            f"""Error when creating the corpus:
+        raise RuntimeError(f"""Error when creating the corpus:
             {str(e)}
-            Are all input files valid CoNLL-U files?"""
-        )
+            Are all input files valid CoNLL-U files?""")
     return c
 
 

{lcpcli-0.3.0 → lcpcli-0.3.1}/lcpcli/corpert.py

@@ -4,7 +4,7 @@ import re
 
 from .cli import _parse_cmd_line
 from .conllu_builder import process_files
-from .utils import default_json, find_config_file, yes_no_input
+from .utils import default_json, find_config_file, say_yes
 
 ERROR_MSG = """
 Unrecognized input format.
@@ -21,6 +21,7 @@ class Corpert:
         output=None,
         extension=None,
         combine=True,
+        force_yes=False,
         **kwargs,
     ):
         """
@@ -39,6 +40,7 @@ class Corpert:
         self._path = os.path.normpath(content)
         self._combine = combine
         self._on_disk = True
+        self._force_yes = force_yes
         if os.path.isfile(content):
             self._input_files.append(content)
         elif os.path.isdir(content):
@@ -75,7 +77,7 @@ class Corpert:
             print(
                 f"The destination folder {self.output} contains some JSON and/or CSV files which this operation might overwrite. Do you want to proceed?"
             )
-            if not yes_no_input():
+            if not say_yes(auto=self._force_yes):
                 print("Aborting the conversion operation.")
                 return
 
@@ -118,7 +120,7 @@ class Corpert:
                 f"The input folder ({self._path}) contains both files with a CoNLL extension and files with a different extension."
             )
             print("Ignore the files with a non-CoNLL extension?")
-            if yes_no_input():
+            if say_yes(auto=self._force_yes):
                 doc_files = [
                     f for f in doc_files if f.lower().endswith((".conll", ".conllu"))
                 ]