granuscore 1.0.0__py3-none-any.whl

granuscore/text_cleaner.py

@@ -0,0 +1,61 @@
+import re
+
+
+class TextCleaner:
+    """Utilities for cleaning and normalizing text."""
+
+    # Regex patterns
+    LATEX_PATTERN = re.compile(
+        r"<\s*l\s*a\s*t\s*e\s*x\s*i\s*t(?:\s+[^>]*)?>.*?<\s*/\s*l\s*a\s*t\s*e\s*x\s*i\s*t\s*>",
+        re.DOTALL | re.IGNORECASE,
+    )
+    ARXIV_META_PATTERN = re.compile(r"arXiv:\d+\.\d+v\d+", re.IGNORECASE)
+    FIGURE_CAPTION_PATTERN = re.compile(r"(Figure|Fig\.)\s*\d+[:].*", re.IGNORECASE)
+    BRACKETS_PATTERN = re.compile(r"\([^)]*\)|\[[^\]]*\]")
+    SINGLE_LETTER_NOISE_PATTERN = re.compile(r"(?:\b[A-Za-z0-9]\b\s+){2,}\b[A-Za-z0-9]\b")
+
+    @classmethod
+    def clean(
+            cls,
+            text: str,
+            *,
+            skip_brackets: bool = False,
+            skip_latex: bool = False,
+            skip_arxiv_meta: bool = False,
+            skip_figures: bool = False,
+            skip_pdf_noise: bool = False,
+    ) -> str:
+        """
+        Clean text by removing various unwanted patterns.
+
+        Args:
+            text: Input text to clean
+            skip_brackets: Remove content in brackets/parentheses
+            skip_latex: Remove LaTeX blocks
+            skip_arxiv_meta: Remove arXiv metadata
+            skip_figures: Remove figure captions
+            skip_pdf_noise: Remove PDF extraction noise
+        Returns:
+            Cleaned text
+        """
+        if skip_latex:
+            text = cls.LATEX_PATTERN.sub(" ", text)
+        if skip_arxiv_meta:
+            text = cls.ARXIV_META_PATTERN.sub(" ", text)
+        if skip_figures:
+            text = cls.FIGURE_CAPTION_PATTERN.sub(" ", text)
+        if skip_brackets:
+            text = cls.BRACKETS_PATTERN.sub(" ", text)
+        if skip_pdf_noise:
+            text = cls.SINGLE_LETTER_NOISE_PATTERN.sub(" ", text)
+
+        # Normalize spacing
+        text = re.sub(r"([.!?])([A-Za-z])", r"\1 \2", text)
+        text = re.sub(r"\s+", " ", text)
+        return text.strip()
+
+    @staticmethod
+    def normalize_spacing(text: str) -> str:
+        """Ensure proper spacing after sentence-ending punctuation."""
+        return re.sub(r"([.!?])([A-Za-z])", r"\1 \2", text)
+

granuscore/utils.py

@@ -0,0 +1,48 @@
+import os
+from multiprocessing.pool import ThreadPool
+from typing import Sequence, Iterator, TypeVar, Callable, Any
+
+import torch
+from spacy.util import is_package
+from tqdm import tqdm
+
+T = TypeVar("T")
+
+
+def ensure_spacy_model(name: str = "en_core_web_sm") -> None:
+    if is_package(name):
+        return
+
+    from spacy.cli import download
+    download(name)
+
+
+def iter_batches(items: Sequence[T], batch_size: int) -> Iterator[Sequence[T]]:
+    for i in range(0, len(items), batch_size):
+        yield items[i: i + batch_size]
+
+
+def get_best_device():
+    if torch.cuda.is_available():
+        return torch.device("cuda")
+    if torch.backends.mps.is_available():
+        return torch.device("mps")
+    return torch.device("cpu")
+
+
+def map_with_progress(
+    f: Callable,
+    xs: list[Any],
+    num_threads: int = os.cpu_count() or 10,
+    pbar: bool = False,
+):
+    """
+    Apply f to each element of xs, using a ThreadPool, and show progress.
+    """
+    pbar_fn = tqdm if pbar else lambda x, *args, **kwargs: x
+
+    if os.getenv("debug"):
+        return list(map(f, pbar_fn(xs, total=len(xs))))
+    else:
+        with ThreadPool(min(num_threads, len(xs))) as pool:
+            return list(pbar_fn(pool.imap(f, xs), total=len(xs)))

granuscore-1.0.0.dist-info/METADATA

@@ -0,0 +1,242 @@
+Metadata-Version: 2.4
+Name: granuscore
+Version: 1.0.0
+Summary: Granularity scoring for natural language
+Author-email: Lukas Ellinger <lukas.ellinger@tum.de>
+License-Expression: MIT
+Project-URL: Repository, https://github.com/lukasellinger/granuscore
+Project-URL: Issues, https://github.com/lukasellinger/granuscore/issues
+Keywords: granularity,natural language processing,nlp,text analysis,semantic abstraction,specificity
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: <3.13,>=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: faiss-cpu>=1.13.2
+Requires-Dist: hierarchy_transformers>=0.1.1
+Requires-Dist: lightgbm>=4.6.0
+Requires-Dist: spacy>=3.8.11
+Requires-Dist: torch>=2.9.1
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: platformdirs>=3.10.0
+Requires-Dist: requests>=2.31.0
+Requires-Dist: tqdm>=4.66.0
+Requires-Dist: openai>=2.21.0
+Provides-Extra: dev
+Requires-Dist: optuna>=4.6.0; extra == "dev"
+Requires-Dist: pygam>=0.12.0; extra == "dev"
+Requires-Dist: python-dotenv>=1.2.1; extra == "dev"
+Requires-Dist: ollama>=0.6.1; extra == "dev"
+Requires-Dist: google-genai>=1.66.0; extra == "dev"
+Requires-Dist: wget>=3.2; extra == "dev"
+Dynamic: license-file
+
+[![PyPI version](https://img.shields.io/pypi/v/granuscore.svg)](https://pypi.org/project/granuscore/)
+[![GitHub](https://img.shields.io/badge/GitHub-Repository-black)](https://github.com/lukasellinger/granuscore)
+# Granuscore
+
+**Granuscore** is a Python library for measuring the *semantic granularity* of natural language text.
+
+It provides an end-to-end pipeline that:
+
+1. splits text into referential units,
+2. assigns continuous granularity scores to each unit,
+3. aggregates these scores into document-level estimates.
+
+Granuscore is designed for analyzing how *fine-grained* or *coarse-grained* textual expressions are in applications such as question answering, educational dialogue, summarization, and scientific writing.
+
+---
+
+## Installation
+
+Install from PyPI:
+
+```bash
+pip install granuscore
+```
+
+Or install the latest development version locally:
+
+```bash
+git clone https://github.com/lukasellinger/granuscore.git
+cd granuscore
+pip install -e .
+```
+
+Optional development dependencies:
+
+```bash
+pip install -e ".[dev]"
+```
+
+---
+
+## Quick Start
+
+```python
+from granuscore import GranuScore
+
+scorer = GranuScore()
+
+text = """
+Tony Hawk was born in San Diego.
+"""
+
+score = scorer(text)
+
+print(score)
+```
+
+By default, Granuscore returns percentile scores, where higher values correspond to coarser-grained expressions.
+
+---
+
+## Default Configuration
+
+The default configuration reproduces the setup used in the paper.
+
+```python
+scorer = GranuScore()
+```
+
+Equivalent to:
+
+```python
+scorer = GranuScore(
+    predictor_type="hit",
+)
+```
+
+Default settings:
+
+- `predictor_type="hit"`
+- `model_name="Hierarchy-Transformers/HiT-MiniLM-L12-WordNetNoun"`
+- `search_method="random_anchors"`
+- `random_anchors_k=999`
+
+Required artifacts such as:
+- FAISS indices,
+- anchor vectors,
+- LightGBM models,
+- and reference percentile distributions
+
+are automatically downloaded and cached on first use.
+
+---
+
+## Important Compatibility Note
+
+The default configuration works out of the box and is the recommended setup.
+
+If you customize components such as:
+- the embedding model,
+- search method,
+- FAISS index,
+- anchor vectors,
+- or LightGBM model,
+
+you must ensure that all resources are compatible with each other.
+
+For example, a LightGBM model trained using:
+
+```python
+search_method="random_anchors"
+```
+
+should not be combined with:
+
+```python
+search_method="nearest_neighbor"
+```
+
+Similarly, FAISS indices, anchor vectors, percentile reference distributions, and LightGBM models must originate from the same embedding space and training configuration.
+
+Compatibility between custom resources is not validated automatically.
+
+---
+
+## Notebook Tutorial
+
+An interactive introduction is available in:
+
+```text
+notebooks/getting_started.ipynb
+```
+
+---
+
+## Repository Structure
+
+```text
+granuscore/
+├── src/
+│   └── granuscore/
+│       ├── pipeline.py
+│       ├── granularity_predictor.py
+│       ├── claim_splitter.py
+│       ├── bucket_output.py
+│       ├── cache.py
+│       └── artifacts.py
+├── notebooks/
+│   └── getting_started.ipynb
+├── training_scripts/
+├── evaluation/
+├── assets/
+├── data/ (needs to be externally downloaded)
+├── pyproject.toml
+├── LICENSE
+└── README.md
+```
+
+---
+
+## Reproducing Paper Experiments
+
+The datasets and precomputed resources required to reproduce the experiments from the paper are available here:
+
+https://drive.google.com/drive/folders/1mJdUENOxHEiuYn-_f1KRQ1PZggXJDnb4?usp=sharing
+
+Download the archive and extract it into the repository root:
+
+```bash
+unzip data.zip
+```
+
+This will create the expected directory structure used by the training and evaluation scripts.
+
+---
+
+## Training Pipeline
+
+Training uses precomputed `.pkl` feature files.
+
+1. Generate precomputed datasets:
+
+```text
+training_scripts/build_precalc_data/
+```
+
+2. Train LightGBM models:
+
+```bash
+python training_scripts/train_lgb_models.py
+```
+
+---
+
+## Citation
+Updated Citation information will be added after publication.
+
+```bibtex
+@misc{ellinger2026granuscore,
+  title={Granuscore: A Reference-Free Measure of Granularity for Text Analysis and Question Answering},
+  author={Ellinger, Lukas and Fichtl, Alexander M. and Anschütz, Miriam and Groh, Georg},
+  year={2026}
+}
+```

granuscore-1.0.0.dist-info/RECORD

@@ -0,0 +1,16 @@
+granuscore/__init__.py,sha256=nG10Jgicc_EngbTgxXyyNLL3M3GF6NOWMNmUYytKVdU,268
+granuscore/artifcats.py,sha256=BgCatJuAgkERcfV6f_xS0NqyjsawOlhHrsrrm1Ce7vA,3973
+granuscore/bucket_output.py,sha256=y9yiHOvhEXW-KwWtRIcdeal7uBj30rkLZIemYUZ8Sas,1633
+granuscore/cache.py,sha256=HSPMMPDO0Dyi3dYwjXnuZ1P5szJeOpuaMXB6M15-2-c,6873
+granuscore/claim_splitter.py,sha256=nub3fZj7E23Y4QvWrQcIC0hETO1ZlaaBVT-R7O9P5TQ,9785
+granuscore/default_assets.py,sha256=A4zGxQ1AqheT3ja8naDR1f_RTaPfD93pOG82sRMfiNA,1287
+granuscore/granularity_predictor.py,sha256=kzGGMHimedCTfHzVL4kz0lnDFvc7ZZvAqXm7_orWsro,25609
+granuscore/loader.py,sha256=5I9Fd9phOszD-od7kVN7RiMycJXRIrFaGdyxmVs59jo,3059
+granuscore/pipeline.py,sha256=apcFHLR2YGfZiXXThSsZqCeNNMDYd7rOADhmQb5-FTU,24396
+granuscore/text_cleaner.py,sha256=nNfy2XUANgKhReM-occwut4LtxXfpe0s7fp-TGBZLwA,2109
+granuscore/utils.py,sha256=h-yNEYA1zqazd0kAjjtERojH3vXHVZzAu8Jxh7B5rDM,1237
+granuscore-1.0.0.dist-info/licenses/LICENSE,sha256=Sev5Z9X5Afo9ThLkUqBEevqX8ERAoFEn6rvEtQjtPwU,1075
+granuscore-1.0.0.dist-info/METADATA,sha256=1cp9QnCHpwqLiStf41bpoOhl6hfegRtEfo7whDza_Mc,5882
+granuscore-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+granuscore-1.0.0.dist-info/top_level.txt,sha256=BWgkEbJfbqBpAJdIrF2uXwelo7UuMlJkvyQc9xA0glg,11
+granuscore-1.0.0.dist-info/RECORD,,

granuscore-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

granuscore-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Granuscore Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

granuscore-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+granuscore