udsearch 0.1.0__tar.gz

udsearch-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024-2026 Furkan Akkurt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

udsearch-0.1.0/PKG-INFO

@@ -0,0 +1,140 @@
+Metadata-Version: 2.4
+Name: udsearch
+Version: 0.1.0
+Summary: Search, match, and batch-edit Universal Dependencies treebanks
+Author-email: Furkan Akkurt <furkan.akkurt@bogazici.edu.tr>
+License: MIT
+Project-URL: Homepage, https://gitlab.com/furkan4829/tools/ud-tools
+Project-URL: Issues, https://gitlab.com/furkan4829/tools/ud-tools/-/issues
+Keywords: universal-dependencies,treebank,conllu,nlp,linguistics,search
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# udsearch
+
+Search, match, and batch-edit [Universal Dependencies](https://universaldependencies.org/) treebanks from the command line or as a Python library.
+
+Pure Python, no external dependencies.
+
+## Install
+
+```bash
+pip install udsearch
+```
+
+## CLI
+
+```bash
+# Search for tokens
+udsearch "UPOS=NOUN & Case=Dat" -t Turkish-BOUN
+udsearch "UPOS=NOUN|PROPN & deprel=nsubj" -f corpus.conllu
+
+# Structural patterns (multi-node)
+udsearch $'v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v' -f corpus.conllu
+udsearch --pattern-file query.txt -f corpus.conllu
+
+# Clustering (Grew-match style)
+udsearch "UPOS=NOUN" -t Turkish-BOUN --cluster
+udsearch "deprel=obl" -t Turkish-BOUN --cluster Case Number
+
+# Batch rewrite (dry-run by default)
+udsearch "UPOS=NOUN & lemma=yok" --set "UPOS=ADJ" --set "Polarity=Neg" -f tb.conllu
+udsearch "UPOS=NOUN & lemma=yok" --set "UPOS=ADJ" -f tb.conllu --apply
+
+# Structural rewrite (target specific nodes)
+udsearch $'v: [UPOS=VERB]\ns: [] -nsubj-> v' --set "s.Case=Nom" -f tb.conllu --apply
+
+# Treebank management
+udsearch --list tr           # list Turkish treebanks on GitHub
+udsearch --list-cached       # show downloaded treebanks
+```
+
+## Pattern syntax
+
+### Single-node
+
+```
+UPOS=NOUN                    exact match
+UPOS=NOUN|PROPN              alternatives
+lemma=/^yap/                 regex
+!PronType=Prs                negation
+PronType                     feature exists
+UPOS=NOUN & Case=Dat         conjunction
+```
+
+### Structural (multi-node)
+
+```
+v: [UPOS=VERB]                             named node
+s: [UPOS=PRON & Case=Nom] -nsubj-> v      dependency relation
+!a: [UPOS=AUX] -aux-> v                   negated (must NOT exist)
+d: [] -nsubj|obj-> v                       deprel alternatives
+d: [] -/^nsubj/-> v                        deprel regex
+d: [] -> v                                 any relation
+s << v                                     linear precedence
+```
+
+### Rewrite operations
+
+```
+Polarity=Neg                 add/set feature
+UPOS=ADJ                    change column field
+-Case                        remove feature
+MISC.Lang=en                 set MISC field
+s.Case=Nom                   target node in structural pattern
+```
+
+## Library usage
+
+```python
+from udsearch import parse_conllu, parse_pattern, search_treebank
+from udsearch import parse_structural, match_structural
+
+# Parse CoNLL-U
+sentences = parse_conllu(open("corpus.conllu").read())
+
+# Single-node search
+pattern = parse_pattern("UPOS=VERB & Tense=Past")
+for sent, tokens in search_treebank(sentences, pattern):
+    print(sent.sent_id, [t.form for t in tokens])
+
+# Structural search
+sp = parse_structural("v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v")
+for sent, bindings in search_structural(sentences, sp):
+    for b in bindings:
+        print(f"{b['v'].form} <- {b['s'].form}")
+
+# Batch rewrite
+from udsearch import apply_operations, parse_set_operations
+ops = parse_set_operations(["Polarity=Neg"])
+for sent, tokens in search_treebank(sentences, pattern):
+    for token in tokens:
+        changes = apply_operations(token, ops)
+```
+
+### Dict-based API (for web apps)
+
+```python
+from udsearch import match_structural_dicts, apply_operations_to_dicts
+
+# Works with dict-based wordlines (e.g., from a database)
+wordlines = [{"id_f": "1", "form": "cat", "upos": "NOUN", ...}, ...]
+matches = match_structural_dicts("UPOS=NOUN", wordlines)
+modified, changes = apply_operations_to_dicts(wordlines, "UPOS=NOUN", ["Case=Acc"])
+```
+
+## License
+
+MIT

udsearch-0.1.0/README.md

@@ -0,0 +1,116 @@
+# udsearch
+
+Search, match, and batch-edit [Universal Dependencies](https://universaldependencies.org/) treebanks from the command line or as a Python library.
+
+Pure Python, no external dependencies.
+
+## Install
+
+```bash
+pip install udsearch
+```
+
+## CLI
+
+```bash
+# Search for tokens
+udsearch "UPOS=NOUN & Case=Dat" -t Turkish-BOUN
+udsearch "UPOS=NOUN|PROPN & deprel=nsubj" -f corpus.conllu
+
+# Structural patterns (multi-node)
+udsearch $'v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v' -f corpus.conllu
+udsearch --pattern-file query.txt -f corpus.conllu
+
+# Clustering (Grew-match style)
+udsearch "UPOS=NOUN" -t Turkish-BOUN --cluster
+udsearch "deprel=obl" -t Turkish-BOUN --cluster Case Number
+
+# Batch rewrite (dry-run by default)
+udsearch "UPOS=NOUN & lemma=yok" --set "UPOS=ADJ" --set "Polarity=Neg" -f tb.conllu
+udsearch "UPOS=NOUN & lemma=yok" --set "UPOS=ADJ" -f tb.conllu --apply
+
+# Structural rewrite (target specific nodes)
+udsearch $'v: [UPOS=VERB]\ns: [] -nsubj-> v' --set "s.Case=Nom" -f tb.conllu --apply
+
+# Treebank management
+udsearch --list tr           # list Turkish treebanks on GitHub
+udsearch --list-cached       # show downloaded treebanks
+```
+
+## Pattern syntax
+
+### Single-node
+
+```
+UPOS=NOUN                    exact match
+UPOS=NOUN|PROPN              alternatives
+lemma=/^yap/                 regex
+!PronType=Prs                negation
+PronType                     feature exists
+UPOS=NOUN & Case=Dat         conjunction
+```
+
+### Structural (multi-node)
+
+```
+v: [UPOS=VERB]                             named node
+s: [UPOS=PRON & Case=Nom] -nsubj-> v      dependency relation
+!a: [UPOS=AUX] -aux-> v                   negated (must NOT exist)
+d: [] -nsubj|obj-> v                       deprel alternatives
+d: [] -/^nsubj/-> v                        deprel regex
+d: [] -> v                                 any relation
+s << v                                     linear precedence
+```
+
+### Rewrite operations
+
+```
+Polarity=Neg                 add/set feature
+UPOS=ADJ                    change column field
+-Case                        remove feature
+MISC.Lang=en                 set MISC field
+s.Case=Nom                   target node in structural pattern
+```
+
+## Library usage
+
+```python
+from udsearch import parse_conllu, parse_pattern, search_treebank
+from udsearch import parse_structural, match_structural
+
+# Parse CoNLL-U
+sentences = parse_conllu(open("corpus.conllu").read())
+
+# Single-node search
+pattern = parse_pattern("UPOS=VERB & Tense=Past")
+for sent, tokens in search_treebank(sentences, pattern):
+    print(sent.sent_id, [t.form for t in tokens])
+
+# Structural search
+sp = parse_structural("v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v")
+for sent, bindings in search_structural(sentences, sp):
+    for b in bindings:
+        print(f"{b['v'].form} <- {b['s'].form}")
+
+# Batch rewrite
+from udsearch import apply_operations, parse_set_operations
+ops = parse_set_operations(["Polarity=Neg"])
+for sent, tokens in search_treebank(sentences, pattern):
+    for token in tokens:
+        changes = apply_operations(token, ops)
+```
+
+### Dict-based API (for web apps)
+
+```python
+from udsearch import match_structural_dicts, apply_operations_to_dicts
+
+# Works with dict-based wordlines (e.g., from a database)
+wordlines = [{"id_f": "1", "form": "cat", "upos": "NOUN", ...}, ...]
+matches = match_structural_dicts("UPOS=NOUN", wordlines)
+modified, changes = apply_operations_to_dicts(wordlines, "UPOS=NOUN", ["Case=Acc"])
+```
+
+## License
+
+MIT

udsearch-0.1.0/pyproject.toml

@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=68.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "udsearch"
+version = "0.1.0"
+description = "Search, match, and batch-edit Universal Dependencies treebanks"
+requires-python = ">=3.10"
+license = {text = "MIT"}
+authors = [
+    {name = "Furkan Akkurt", email = "furkan.akkurt@bogazici.edu.tr"},
+]
+readme = "README.md"
+keywords = ["universal-dependencies", "treebank", "conllu", "nlp", "linguistics", "search"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Text Processing :: Linguistic",
+]
+
+[project.urls]
+Homepage = "https://gitlab.com/furkan4829/tools/ud-tools"
+Issues = "https://gitlab.com/furkan4829/tools/ud-tools/-/issues"
+
+[project.scripts]
+udsearch = "udsearch.cli:main_search"
+
+[tool.setuptools]
+packages = ["udsearch"]
+
+[tool.ruff]
+line-length = 120

udsearch-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

udsearch-0.1.0/tests/test_compat.py

@@ -0,0 +1,364 @@
+"""Tests for dict-based compatibility layer (BoAT integration)."""
+
+from udsearch._compat import (
+    apply_operations_to_dicts,
+    dict_from_token,
+    match_structural_dicts,
+    sentence_from_wordlines,
+    token_from_dict,
+)
+from udsearch.conllu import Token
+
+
+# Sample wordline dicts in BoAT format
+def _sample_wordlines():
+    """A simple sentence: 'The cat sat.'"""
+    return [
+        {
+            "id_f": "1",
+            "form": "The",
+            "lemma": "the",
+            "upos": "DET",
+            "xpos": "_",
+            "feats": "Definite=Def|PronType=Art",
+            "head": "2",
+            "deprel": "det",
+            "deps": "_",
+            "misc": "_",
+        },
+        {
+            "id_f": "2",
+            "form": "cat",
+            "lemma": "cat",
+            "upos": "NOUN",
+            "xpos": "_",
+            "feats": "Case=Nom|Number=Sing",
+            "head": "3",
+            "deprel": "nsubj",
+            "deps": "_",
+            "misc": "_",
+        },
+        {
+            "id_f": "3",
+            "form": "sat",
+            "lemma": "sit",
+            "upos": "VERB",
+            "xpos": "_",
+            "feats": "Mood=Ind|Tense=Past|VerbForm=Fin",
+            "head": "0",
+            "deprel": "root",
+            "deps": "_",
+            "misc": "_",
+        },
+        {
+            "id_f": "4",
+            "form": ".",
+            "lemma": ".",
+            "upos": "PUNCT",
+            "xpos": "_",
+            "feats": "_",
+            "head": "3",
+            "deprel": "punct",
+            "deps": "_",
+            "misc": "SpaceAfter=No",
+        },
+    ]
+
+
+def _two_sentence_wordlines():
+    """Two sentences for multi-sentence testing."""
+    s1 = _sample_wordlines()
+    s2 = [
+        {
+            "id_f": "1",
+            "form": "Dogs",
+            "lemma": "dog",
+            "upos": "NOUN",
+            "xpos": "_",
+            "feats": "Case=Nom|Number=Plur",
+            "head": "2",
+            "deprel": "nsubj",
+            "deps": "_",
+            "misc": "_",
+        },
+        {
+            "id_f": "2",
+            "form": "run",
+            "lemma": "run",
+            "upos": "VERB",
+            "xpos": "_",
+            "feats": "Mood=Ind|Tense=Pres|VerbForm=Fin",
+            "head": "0",
+            "deprel": "root",
+            "deps": "_",
+            "misc": "SpaceAfter=No",
+        },
+        {
+            "id_f": "3",
+            "form": ".",
+            "lemma": ".",
+            "upos": "PUNCT",
+            "xpos": "_",
+            "feats": "_",
+            "head": "2",
+            "deprel": "punct",
+            "deps": "_",
+            "misc": "_",
+        },
+    ]
+    return s1, s2
+
+
+# ---------------------------------------------------------------------------
+# token_from_dict / dict_from_token
+# ---------------------------------------------------------------------------
+
+
+class TestTokenConversion:
+    def test_roundtrip(self):
+        wl = _sample_wordlines()[1]  # cat
+        token = token_from_dict(wl)
+        assert isinstance(token, Token)
+        assert token.id == "2"
+        assert token.form == "cat"
+        assert token.upos == "NOUN"
+        assert token.feats == {"Case": "Nom", "Number": "Sing"}
+        assert token.head == "3"
+        assert token.deprel == "nsubj"
+
+        # Convert back
+        result = dict_from_token(token)
+        assert result == wl
+
+    def test_underscore_fields(self):
+        wl = _sample_wordlines()[0]  # The — xpos=_, misc=_
+        token = token_from_dict(wl)
+        assert token.xpos == "_"
+        assert token.misc == {}
+        result = dict_from_token(token)
+        assert result["misc"] == "_"
+
+    def test_misc_field(self):
+        wl = _sample_wordlines()[3]  # . — misc=SpaceAfter=No
+        token = token_from_dict(wl)
+        assert token.misc == {"SpaceAfter": "No"}
+        result = dict_from_token(token)
+        assert result["misc"] == "SpaceAfter=No"
+
+
+# ---------------------------------------------------------------------------
+# sentence_from_wordlines
+# ---------------------------------------------------------------------------
+
+
+class TestSentenceFromWordlines:
+    def test_basic(self):
+        wls = _sample_wordlines()
+        sent = sentence_from_wordlines(wls, sent_id="test-1", text="The cat sat.")
+        assert sent.sent_id == "test-1"
+        assert sent.text == "The cat sat."
+        assert len(sent.tokens) == 4
+        assert sent.tokens[0].form == "The"
+        assert sent.tokens[1].upos == "NOUN"
+
+    def test_mwt_preserved(self):
+        """MWT range tokens are preserved in _all_lines but excluded from tokens."""
+        wls = [
+            {
+                "id_f": "1-2",
+                "form": "Gidiyorum",
+                "lemma": "_",
+                "upos": "_",
+                "xpos": "_",
+                "feats": "_",
+                "head": "_",
+                "deprel": "_",
+                "deps": "_",
+                "misc": "SpaceAfter=No",
+            },
+            {
+                "id_f": "1",
+                "form": "Gidiyor",
+                "lemma": "git",
+                "upos": "VERB",
+                "xpos": "_",
+                "feats": "Aspect=Prog",
+                "head": "0",
+                "deprel": "root",
+                "deps": "_",
+                "misc": "_",
+            },
+            {
+                "id_f": "2",
+                "form": "um",
+                "lemma": "ben",
+                "upos": "PRON",
+                "xpos": "_",
+                "feats": "Case=Nom",
+                "head": "1",
+                "deprel": "nsubj",
+                "deps": "_",
+                "misc": "_",
+            },
+        ]
+        sent = sentence_from_wordlines(wls, sent_id="mwt", text="Gidiyorum.")
+        # MWT excluded from searchable tokens
+        assert len(sent.tokens) == 2
+        assert sent.tokens[0].form == "Gidiyor"
+
+
+# ---------------------------------------------------------------------------
+# match_structural_dicts
+# ---------------------------------------------------------------------------
+
+
+class TestMatchStructuralDicts:
+    def test_basic_search(self):
+        wls = _sample_wordlines()
+        results = match_structural_dicts("UPOS=NOUN", wls)
+        assert len(results) == 1
+        assert results[0]["target"]["form"] == "cat"
+
+    def test_structural_pattern_string(self):
+        wls = _sample_wordlines()
+        results = match_structural_dicts(
+            "v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v",
+            wls,
+        )
+        assert len(results) == 1
+        assert results[0]["v"]["form"] == "sat"
+        assert results[0]["s"]["form"] == "cat"
+
+    def test_negated_node(self):
+        wls = _sample_wordlines()
+        # VERB without advmod
+        results = match_structural_dicts(
+            "v: [UPOS=VERB & deprel=root]\n!a: [] -advmod-> v",
+            wls,
+        )
+        assert len(results) == 1
+        assert results[0]["v"]["form"] == "sat"
+
+    def test_no_match(self):
+        wls = _sample_wordlines()
+        results = match_structural_dicts("UPOS=NUM", wls)
+        assert results == []
+
+    def test_result_is_dict(self):
+        """Results should be dicts, not Token objects."""
+        wls = _sample_wordlines()
+        results = match_structural_dicts("UPOS=NOUN", wls)
+        binding = results[0]
+        assert isinstance(binding["target"], dict)
+        assert "id_f" in binding["target"]
+        assert "form" in binding["target"]
+
+    def test_with_parsed_structural_pattern(self):
+        from udsearch.structural import parse_structural
+
+        wls = _sample_wordlines()
+        pattern = parse_structural("v: [UPOS=VERB]\ns: [] -nsubj-> v")
+        results = match_structural_dicts(pattern, wls)
+        assert len(results) == 1
+
+
+# ---------------------------------------------------------------------------
+# apply_operations_to_dicts
+# ---------------------------------------------------------------------------
+
+
+class TestApplyOperationsToDicts:
+    def test_simple_rewrite(self):
+        wls = _sample_wordlines()
+        modified, changes = apply_operations_to_dicts(
+            wls,
+            pattern="UPOS=NOUN",
+            operations=["Number=Plur"],
+        )
+        # Only cat (NOUN) should be modified
+        assert len(changes) == 1
+        assert changes[0]["token_id"] == "2"
+        assert changes[0]["form"] == "cat"
+        assert "Number: Sing → Plur" in changes[0]["descriptions"][0]
+
+        # Check modified wordlines
+        cat_wl = next(wl for wl in modified if wl["id_f"] == "2")
+        assert "Number=Plur" in cat_wl["feats"]
+        assert "Number=Sing" not in cat_wl["feats"]
+
+        # Other tokens unchanged
+        the_wl = next(wl for wl in modified if wl["id_f"] == "1")
+        assert the_wl == _sample_wordlines()[0]
+
+    def test_structural_rewrite_node_targeted(self):
+        wls = _sample_wordlines()
+        modified, changes = apply_operations_to_dicts(
+            wls,
+            pattern="v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v",
+            operations=["s.Case=Acc"],  # target the nsubj node
+        )
+        assert len(changes) == 1
+        assert changes[0]["node_name"] == "s"
+        assert changes[0]["token_id"] == "2"
+        assert "Case: Nom → Acc" in changes[0]["descriptions"][0]
+
+        cat_wl = next(wl for wl in modified if wl["id_f"] == "2")
+        assert "Case=Acc" in cat_wl["feats"]
+
+    def test_structural_rewrite_default_ops(self):
+        """Default ops (no node prefix) apply to anchor node."""
+        wls = _sample_wordlines()
+        modified, changes = apply_operations_to_dicts(
+            wls,
+            pattern="v: [UPOS=VERB]\ns: [UPOS=NOUN] -nsubj-> v",
+            operations=["Polarity=Neg"],  # no prefix -> anchor (v)
+        )
+        assert len(changes) == 1
+        assert changes[0]["node_name"] == "v"
+        assert changes[0]["token_id"] == "3"
+
+        verb_wl = next(wl for wl in modified if wl["id_f"] == "3")
+        assert "Polarity=Neg" in verb_wl["feats"]
+
+    def test_multiple_operations(self):
+        wls = _sample_wordlines()
+        modified, changes = apply_operations_to_dicts(
+            wls,
+            pattern="UPOS=NOUN",
+            operations=["UPOS=PROPN", "Number=Plur"],
+        )
+        assert len(changes) == 1
+        cat_wl = next(wl for wl in modified if wl["id_f"] == "2")
+        assert cat_wl["upos"] == "PROPN"
+        assert "Number=Plur" in cat_wl["feats"]
+
+    def test_no_match_no_changes(self):
+        wls = _sample_wordlines()
+        modified, changes = apply_operations_to_dicts(
+            wls,
+            pattern="UPOS=NUM",
+            operations=["Number=Plur"],
+        )
+        assert changes == []
+        assert modified == _sample_wordlines()
+
+    def test_remove_feature(self):
+        wls = _sample_wordlines()
+        modified, changes = apply_operations_to_dicts(
+            wls,
+            pattern="UPOS=NOUN",
+            operations=["-Case"],
+        )
+        assert len(changes) == 1
+        cat_wl = next(wl for wl in modified if wl["id_f"] == "2")
+        assert "Case" not in cat_wl["feats"]
+        assert "Number=Sing" in cat_wl["feats"]
+
+    def test_preserves_wordline_count(self):
+        wls = _sample_wordlines()
+        modified, _ = apply_operations_to_dicts(
+            wls,
+            pattern="UPOS=NOUN",
+            operations=["Number=Plur"],
+        )
+        assert len(modified) == len(wls)