hyperbase-parser-ab 0.1.0__tar.gz → 0.2.0__tar.gz

hyperbase_parser_ab-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.8
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format

{hyperbase_parser_ab-0.1.0 → hyperbase_parser_ab-0.2.0}/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [0.2.0] - 05-04-2026
+
+### Changed
+
+- Adopted new hyperbase API (0.9.0).
+
 ## [0.1.0] - 02-04-2026 - extracted from graphbrain
 
 ### Added

{hyperbase_parser_ab-0.1.0 → hyperbase_parser_ab-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hyperbase-parser-ab
-Version: 0.1.0
+Version: 0.2.0
 Summary: Semantic Hypergraph AlphaBeta Parser
 Project-URL: Homepage, https://hyperquest.ai/hyperbase
 Author-email: "Telmo Menezes et al." <telmo@telmomenezes.net>
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.10
-Requires-Dist: hyperbase>=0.8.0
+Requires-Dist: hyperbase>=0.9.0
 Requires-Dist: pip
 Requires-Dist: scikit-learn>=1.3.0
 Requires-Dist: spacy>=3.8.0
@@ -24,6 +24,7 @@ Requires-Dist: transformers>=4.46.0
 Provides-Extra: dev
 Requires-Dist: coverage>=7.4.3; extra == 'dev'
 Requires-Dist: datasets>=4.0.0; extra == 'dev'
+Requires-Dist: evaluate>=0.4.6; extra == 'dev'
 Requires-Dist: mypy>=1.8.0; extra == 'dev'
 Requires-Dist: pre-commit>=3.6.2; extra == 'dev'
 Requires-Dist: pytest>=9.0.0; extra == 'dev'

hyperbase_parser_ab-0.2.0/VERSION

@@ -0,0 +1 @@
+0.2.0

{hyperbase_parser_ab-0.1.0 → hyperbase_parser_ab-0.2.0}/pyproject.toml

@@ -26,12 +26,12 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Information Analysis",
 ]
 dependencies = [
-    "hyperbase>=0.8.0",
+    "hyperbase>=0.9.0",
     "scikit-learn>=1.3.0",
     "spacy>=3.8.0",
     "torch>=2.0.0",
     "transformers>=4.46.0",
-    "pip",  # so that spaCy models can be easily installed with uv
+    "pip", # so that spaCy models can be easily installed with uv
 ]
 
 [tool.uv.sources]
@@ -46,6 +46,7 @@ dev = [
     "coverage>=7.4.3",
     "datasets>=4.0.0",
     "pytest>=9.0.0",
+    "evaluate>=0.4.6",
 ]
 
 [project.urls]
@@ -70,3 +71,15 @@ strict = true
 
 [tool.ruff]
 target-version = "py310"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF", "Q", "C4", "PT", "N", "ANN"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["E501", "ANN001", "ANN201", "ANN202", "ANN205", "D100", "D101", "D102", "D400", "D415"]
+
+[tool.ruff.lint.flake8-quotes]
+inline-quotes = "double"
+
+[tool.ruff.format]
+quote-style = "double"

hyperbase_parser_ab-0.2.0/scripts/generate_alpha_training_data.py

@@ -0,0 +1,113 @@
+import argparse
+import json
+
+from hyperbase import hedge
+
+from hyperbase_parser_ab import AlphaBetaParser
+
+if __name__ == "__main__":
+    arg_parser = argparse.ArgumentParser(description="Generate alpha training data.")
+    arg_parser.add_argument("infile", type=str, help="input jsonl file")
+    arg_parser.add_argument("outfile", type=str, help="output tsv file")
+    arg_parser.add_argument(
+        "--lang", type=str, default="en", help="language (default: en)"
+    )
+    args = arg_parser.parse_args()
+
+    total_sentences = 0
+    ignored_sentences = 0
+    failed_parses = 0
+    total_atoms = 0
+
+    parser = AlphaBetaParser(lang=args.lang)
+
+    with open(args.infile) as infile, open(args.outfile, "w") as outfile:
+        for line in infile.readlines():
+            case = json.loads(line)
+            sentence = case["sentence"]
+            atoms = case["atoms"]
+            parses = parser.parse_sentence(sentence)
+            spacy_sentence = next(iter(parser.doc.sents)) if parser.doc else None
+            if not spacy_sentence or not parses:
+                failed_parses += 1
+            elif case["ignore"]:
+                ignored_sentences += 1
+            elif len(atoms) == len(spacy_sentence):
+                total_sentences += 1
+                total_atoms += len(atoms)
+
+                for i in range(len(atoms)):
+                    atom = atoms[i]
+                    token = spacy_sentence[i]
+                    atom_edge = hedge(atom)
+                    if atom_edge is None:
+                        continue
+
+                    word_before = ""
+                    word_after = ""
+                    pos_before = ""
+                    pos_after = ""
+                    tag_before = ""
+                    tag_after = ""
+                    dep_before = ""
+                    dep_after = ""
+                    punct_before = False
+                    punct_after = False
+                    if i > 0:
+                        word_before = str(spacy_sentence[i - 1])
+                        pos_before = spacy_sentence[i - 1].pos_
+                        tag_before = spacy_sentence[i - 1].tag_
+                        dep_before = spacy_sentence[i - 1].dep_
+                        if spacy_sentence[i - 1].pos_ == "PUNCT":
+                            punct_before = True
+                    if i < len(atoms) - 1:
+                        word_after = str(spacy_sentence[i + 1])
+                        pos_after = spacy_sentence[i + 1].pos_
+                        tag_after = spacy_sentence[i + 1].tag_
+                        dep_after = spacy_sentence[i + 1].dep_
+                        if spacy_sentence[i + 1].pos_ == "PUNCT":
+                            punct_after = True
+
+                    head = token.head
+                    is_root = head is None
+                    has_lefts = token.n_lefts > 0
+                    has_rights = token.n_rights > 0
+                    outfile.write(
+                        ("{}" + "\t{}" * 25 + "\n").format(
+                            atom_edge.mtype(),
+                            str(token),
+                            token.pos_,
+                            token.tag_,
+                            token.dep_,
+                            str(head) if head else "",
+                            head.pos_ if head else "",
+                            head.tag_ if head else "",
+                            head.dep_ if head else "",
+                            is_root,
+                            has_lefts,
+                            has_rights,
+                            token.ent_type_,
+                            token.shape_[:2],
+                            word_before,
+                            word_after,
+                            punct_before,
+                            punct_after,
+                            pos_before,
+                            pos_after,
+                            tag_before,
+                            tag_after,
+                            dep_before,
+                            dep_after,
+                            case["correct"],
+                            case["source"],
+                        )
+                    )
+            else:
+                failed_parses += 1
+            print(
+                f"sentences: {total_sentences}; "
+                f"ignored: {ignored_sentences}; "
+                f"failed: {failed_parses}; "
+                f"atoms: {total_atoms}"
+            )
+        print("done.")

{hyperbase_parser_ab-0.1.0 → hyperbase_parser_ab-0.2.0}/scripts/train_atomizer.py

@@ -1,27 +1,28 @@
 import json
 
+import evaluate
 import numpy as np
-from numpy.typing import NDArray
 from datasets import Dataset
+from numpy.typing import NDArray
 from transformers import (
-    AutoTokenizer,
     AutoModelForTokenClassification,
+    AutoTokenizer,
+    Trainer,
     TrainingArguments,
-    Trainer
 )
 
 
 def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
     """Tokenize each sample and align the original token labels
-       to the new subword (tokenized) structure."""
+    to the new subword (tokenized) structure."""
 
     tokenized_outputs = tokenizer(
         examples["tokens"],
         truncation=True,
-        is_split_into_words=True,     # Important for token-based tasks
+        is_split_into_words=True,  # Important for token-based tasks
         return_offsets_mapping=True,  # We'll use this if needed
-        padding="max_length",         # or "longest" / "do_not_pad"
-        max_length=200                # adjust as needed
+        padding="max_length",  # or "longest" / "do_not_pad"
+        max_length=200,  # adjust as needed
     )
 
     labels_aligned: list[list[int]] = []
@@ -31,7 +32,6 @@ def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
         # repeating the label for all subwords of the original token.
         word_ids: list[int | None] = tokenized_outputs.word_ids(batch_index=i)
         label_ids: list[int] = []
-        previous_word_idx: int | None = None
 
         for word_idx in word_ids:
             if word_idx is None:
@@ -39,7 +39,6 @@ def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
                 label_ids.append(-100)
             else:
                 label_ids.append(label_to_id[labels[word_idx]])
-            previous_word_idx = word_idx
 
         labels_aligned.append(label_ids)
 
@@ -52,8 +51,8 @@ def tokenize_and_align_labels(examples: dict[str, list]) -> dict[str, list]:
 
 def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
     """Compute accuracy at the token level (simple example).
-       You can also compute F1, precision, recall, etc. by ignoring
-       the -100 special tokens."""
+    You can also compute F1, precision, recall, etc. by ignoring
+    the -100 special tokens."""
     logits: NDArray
     labels: NDArray
     logits, labels = eval_pred
@@ -62,33 +61,35 @@ def compute_metrics(eval_pred: tuple[NDArray, NDArray]) -> dict[str, float]:
     # Flatten ignoring -100
     true_predictions: list[int] = []
     true_labels: list[int] = []
-    for pred, lab in zip(predictions, labels):
-        for p, l in zip(pred, lab):
-            if l != -100:  # skip special tokens
-                true_predictions.append(p)
-                true_labels.append(l)
+    for pred, lab in zip(predictions, labels, strict=True):
+        for _pred, _lab in zip(
+            pred,
+            lab,
+            strict=False,
+        ):
+            if _lab != -100:  # skip special tokens
+                true_predictions.append(_pred)
+                true_labels.append(_lab)
 
     results: dict[str, float] = accuracy_metric.compute(
-        references=true_labels,
-        predictions=true_predictions
+        references=true_labels, predictions=true_predictions
     )
     return {"accuracy": results["accuracy"]}
 
 
-if __name__ == '__main__':
-    with open("sentences.jsonl", "rt") as f:
+if __name__ == "__main__":
+    with open("sentences.jsonl") as f:
         sentences: list[dict] = [json.loads(line) for line in f]
 
     dataset_dict: dict[str, list] = {
         "tokens": [sentence["words"] for sentence in sentences],
-        "labels": [sentence["types"] for sentence in sentences]
+        "labels": [sentence["types"] for sentence in sentences],
     }
 
     full_dataset: Dataset = Dataset.from_dict(dataset_dict)
 
     max_words: int = max([len(sentence["words"]) for sentence in sentences])
 
-
     labels: set[str] = set()
     for sentence in sentences:
         labels |= set(sentence["types"])
@@ -103,9 +104,10 @@ if __name__ == '__main__':
     print("Num train samples:", len(train_dataset))
     print("Num test samples: ", len(test_dataset))
 
-
     model_checkpoint: str = "distilbert-base-multilingual-cased"
-    tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True, add_prefix_space=True)
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_checkpoint, use_fast=True, add_prefix_space=True
+    )
 
     # Apply to train/test datasets
     train_dataset = train_dataset.map(tokenize_and_align_labels, batched=True)
@@ -123,7 +125,7 @@ if __name__ == '__main__':
         model_checkpoint,
         num_labels=len(labels),
         id2label=id_to_label,
-        label2id=label_to_id
+        label2id=label_to_id,
     )
 
     accuracy_metric = evaluate.load("accuracy")  # type: ignore[attr-defined]
@@ -139,7 +141,7 @@ if __name__ == '__main__':
         weight_decay=0.01,
         logging_dir="./logs",
         logging_steps=10,
-        report_to="none"  # Set to "tensorboard" if you want logs
+        report_to="none",  # Set to "tensorboard" if you want logs
     )
 
     trainer: Trainer = Trainer(
@@ -148,7 +150,7 @@ if __name__ == '__main__':
         train_dataset=train_dataset,
         eval_dataset=test_dataset,
         processing_class=tokenizer,
-        compute_metrics=compute_metrics
+        compute_metrics=compute_metrics,
     )
 
     trainer.train()

{hyperbase_parser_ab-0.1.0 → hyperbase_parser_ab-0.2.0}/src/hyperbase_parser_ab/alpha.py

@@ -8,20 +8,22 @@ from spacy.tokens import Span
 from hyperbase_parser_ab.atomizer import Atomizer
 
 
-class Alpha(object):
-    def __init__(self, cases_str: str | None = None, use_atomizer: bool = False) -> None:
+class Alpha:
+    def __init__(
+        self, cases_str: str | None = None, use_atomizer: bool = False
+    ) -> None:
         if use_atomizer:
             self.atomizer: Atomizer | None = Atomizer()
         elif cases_str:
             self.atomizer = None
 
-            X: list[tuple[str, str, str, str, str]] = []
+            x: list[tuple[str, str, str, str, str]] = []
             y: list[list[str]] = []
 
-            for line in cases_str.strip().split('\n'):
+            for line in cases_str.strip().split("\n"):
                 sline: str = line.strip()
                 if len(sline) > 0:
-                    row: list[str] = sline.strip().split('\t')
+                    row: list[str] = sline.strip().split("\t")
                     true_value: str = row[0]
                     tag: str = row[3]
                     dep: str = row[4]
@@ -30,40 +32,50 @@ class Alpha(object):
                     pos_after: str = row[19]
 
                     y.append([true_value])
-                    X.append((tag, dep, hpos, hdep, pos_after))
+                    x.append((tag, dep, hpos, hdep, pos_after))
 
             if len(y) > 0:
                 self.empty: bool = False
 
-                self.encX: OneHotEncoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False)
-                self.encX.fit(np.array(X))
-                self.ency: OneHotEncoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False)
+                self.encX: OneHotEncoder = OneHotEncoder(
+                    handle_unknown="ignore", sparse_output=False
+                )
+                self.encX.fit(np.array(x))
+                self.ency: OneHotEncoder = OneHotEncoder(
+                    handle_unknown="ignore", sparse_output=False
+                )
                 self.ency.fit(np.array(y))
 
-                X_: NDArray | spmatrix = self.encX.transform(np.array(X))
+                x_: NDArray | spmatrix = self.encX.transform(np.array(x))
                 y_: NDArray | spmatrix = self.ency.transform(np.array(y))
 
-                self.clf: RandomForestClassifier = RandomForestClassifier(random_state=777)
-                self.clf.fit(X_, y_)
+                self.clf: RandomForestClassifier = RandomForestClassifier(
+                    random_state=777
+                )
+                self.clf.fit(x_, y_)
         else:
             self.empty = True
 
-    def predict(self, sentence: Span, features: list[tuple[str, str, str, str, str]]) -> tuple[str, ...] | list[str]:
+    def predict(
+        self, sentence: Span, features: list[tuple[str, str, str, str, str]]
+    ) -> tuple[str, ...] | list[str]:
         if self.atomizer:
             preds: list[tuple[str, str]] = self.atomizer.atomize(
-                sentence=str(sentence),
-                tokens=[str(token) for token in sentence])
+                sentence=str(sentence), tokens=[str(token) for token in sentence]
+            )
             atom_types: list[str] = [pred[1] for pred in preds]
 
             # force known cases
             for i in range(len(atom_types)):
-                if sentence[i].pos_ == 'VERB':
-                    atom_types[i] = 'P'
+                if sentence[i].pos_ == "VERB":
+                    atom_types[i] = "P"
             return atom_types
         else:
             # an empty classifier always predicts 'C'
             if self.empty:
-                return tuple('C' for _ in range(len(features)))
+                return tuple("C" for _ in range(len(features)))
             _features: NDArray | spmatrix = self.encX.transform(np.array(features))
-            preds_arr: NDArray | spmatrix = self.ency.inverse_transform(self.clf.predict(_features))
-            return tuple(pred[0] if pred else 'C' for pred in preds_arr)
+            preds_arr: NDArray | spmatrix = self.ency.inverse_transform(
+                self.clf.predict(_features)
+            )
+            return tuple(pred[0] if pred else "C" for pred in preds_arr)

{hyperbase_parser_ab-0.1.0 → hyperbase_parser_ab-0.2.0}/src/hyperbase_parser_ab/atomizer.py

@@ -1,8 +1,12 @@
 from collections import Counter
 
 import torch
-from transformers import AutoTokenizer, AutoModelForTokenClassification, PreTrainedTokenizerBase, PreTrainedModel
-
+from transformers import (
+    AutoModelForTokenClassification,
+    AutoTokenizer,
+    PreTrainedModel,
+    PreTrainedTokenizerBase,
+)
 
 HF_REPO: str = "hyperquest/atom-classifier"
 
@@ -11,21 +15,21 @@ class Atomizer:
     def __init__(self, model_path: str | None = None) -> None:
         model_id: str = model_path or HF_REPO
         self.model_path: str = model_id
-        self.tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(model_id, use_fast=True)
-        self.model: PreTrainedModel = AutoModelForTokenClassification.from_pretrained(model_id)
+        self.tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained(
+            model_id, use_fast=True
+        )
+        self.model: PreTrainedModel = AutoModelForTokenClassification.from_pretrained(
+            model_id
+        )
         assert self.model.config.id2label
         self.id2label: dict[int, str] = self.model.config.id2label
 
-    def atomize(self,
-                sentence: str,
-                tokens: list[str] | None = None
-               ) -> list[tuple[str, str]]:
+    def atomize(
+        self, sentence: str, tokens: list[str] | None = None
+    ) -> list[tuple[str, str]]:
         # Tokenize the raw sentence and request offsets
         encoded = self.tokenizer(
-            sentence,
-            return_tensors="pt",
-            truncation=True,
-            return_offsets_mapping=True
+            sentence, return_tensors="pt", truncation=True, return_offsets_mapping=True
         )
 
         offset_mapping = encoded.pop("offset_mapping")  # remove so model doesn't see it
@@ -39,7 +43,9 @@ class Atomizer:
 
         if tokens is not None:
             # Map provided tokens to model predictions based on character offsets
-            return self._map_tokens_to_predictions(sentence, tokens, word_ids, pred_ids, offset_mapping)
+            return self._map_tokens_to_predictions(
+                sentence, tokens, word_ids, pred_ids, offset_mapping
+            )
 
         predicted_labels: list[tuple[str, str]] = []
         current_word_id: int | None = None
@@ -79,13 +85,14 @@ class Atomizer:
 
         return predicted_labels
 
-    def _map_tokens_to_predictions(self,
-                                   sentence: str,
-                                   tokens: list[str],
-                                   word_ids: list[int | None],
-                                   pred_ids: list[int],
-                                   offset_mapping: list[list[int]]
-                                  ) -> list[tuple[str, str]]:
+    def _map_tokens_to_predictions(
+        self,
+        sentence: str,
+        tokens: list[str],
+        word_ids: list[int | None],
+        pred_ids: list[int],
+        offset_mapping: list[list[int]],
+    ) -> list[tuple[str, str]]:
         """
         Maps provided tokens to model predictions by finding character offsets
         and assigning the most appropriate label based on overlapping model tokens.
@@ -105,10 +112,10 @@ class Atomizer:
 
         # For each provided token, collect overlapping model predictions
         result: list[tuple[str, str]] = []
-        for token, positions in zip(tokens, token_positions):
+        for token, positions in zip(tokens, token_positions, strict=True):
             if positions is None:
                 # Token not found in sentence - assign default label
-                result.append((token, 'C'))
+                result.append((token, "C"))
                 continue
 
             token_start: int
@@ -133,10 +140,12 @@ class Atomizer:
             # Assign the most common label, or first label if tie
             if overlapping_labels:
                 # Use most common label
-                most_common_label: str = Counter(overlapping_labels).most_common(1)[0][0]
+                most_common_label: str = Counter(overlapping_labels).most_common(1)[0][
+                    0
+                ]
                 result.append((token, most_common_label))
             else:
                 # No overlap found - use default
-                result.append((token, 'C'))
+                result.append((token, "C"))
 
         return result

hyperbase_parser_ab-0.2.0/src/hyperbase_parser_ab/lang_models.py

@@ -0,0 +1,9 @@
+SPACY_MODELS: dict[str, list[str]] = {
+    "de": ["de_dep_news_trf", "de_core_news_lg", "de_core_news_md", "de_core_news_sm"],
+    "en": ["en_core_web_trf", "en_core_web_lg", "en_core_web_md", "en_core_web_sm"],
+    "es": ["es_dep_news_trf", "es_core_news_lg", "es_core_news_md", "es_core_news_sm"],
+    "fr": ["fr_dep_news_trf", "fr_core_news_lg", "fr_core_news_md", "fr_core_news_sm"],
+    "it": ["it_core_news_lg", "it_core_news_md", "it_core_news_sm"],
+    "pt": ["pt_core_news_lg", "pt_core_news_md", "pt_core_news_sm"],
+    "zh": ["zh_core_news_trf", "zh_core_news_lg", "zh_core_news_md", "zh_core_news_sm"],
+}