basa-evals 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- basa_evals/__init__.py +3 -0
- basa_evals/adapters/__init__.py +15 -0
- basa_evals/adapters/basa.py +61 -0
- basa_evals/adapters/base.py +26 -0
- basa_evals/adapters/identity.py +22 -0
- basa_evals/adapters/sastrawi.py +43 -0
- basa_evals/adapters/simple_regex.py +53 -0
- basa_evals/cli.py +563 -0
- basa_evals/configs/suites/downstream_smsa.yaml +16 -0
- basa_evals/configs/suites/pilot_210.yaml +33 -0
- basa_evals/configs/suites/tokenizer_pilot.yaml +27 -0
- basa_evals/core/__init__.py +1 -0
- basa_evals/core/alignment.py +227 -0
- basa_evals/core/gate.py +168 -0
- basa_evals/core/loader.py +109 -0
- basa_evals/core/provenance.py +36 -0
- basa_evals/core/registry.py +42 -0
- basa_evals/core/runner.py +188 -0
- basa_evals/core/schema.py +388 -0
- basa_evals/core/tokenizer.py +91 -0
- basa_evals/data/baselines/simple_regex_dict_v1.json +61 -0
- basa_evals/data/challenge/mixed_challenge_v1.jsonl +30 -0
- basa_evals/data/challenge/negative_controls_code_switching_v1.jsonl +30 -0
- basa_evals/data/challenge/negative_controls_kbbi_v1.jsonl +30 -0
- basa_evals/data/downstream/smsa_test_v1.jsonl +150 -0
- basa_evals/data/gold/abbreviation_phonetic_v1.jsonl +30 -0
- basa_evals/data/gold/repetition_reduction_v1.jsonl +30 -0
- basa_evals/data/gold/slang_lexical_v1.jsonl +30 -0
- basa_evals/data/gold/typo_nonword_v1.jsonl +30 -0
- basa_evals/downstream/__init__.py +9 -0
- basa_evals/downstream/base.py +18 -0
- basa_evals/downstream/hf.py +59 -0
- basa_evals/downstream/mock.py +72 -0
- basa_evals/downstream/registry.py +57 -0
- basa_evals/downstream/runner.py +93 -0
- basa_evals/metrics/__init__.py +29 -0
- basa_evals/metrics/aggregate.py +168 -0
- basa_evals/metrics/downstream.py +167 -0
- basa_evals/metrics/linguistic.py +65 -0
- basa_evals/metrics/preservation.py +31 -0
- basa_evals/metrics/tokenization.py +173 -0
- basa_evals/py.typed +1 -0
- basa_evals/reporters/__init__.py +24 -0
- basa_evals/reporters/console.py +331 -0
- basa_evals/reporters/exporter.py +75 -0
- basa_evals/reporters/markdown.py +223 -0
- basa_evals/tokenizers/__init__.py +13 -0
- basa_evals/tokenizers/base.py +31 -0
- basa_evals/tokenizers/hf_adapter.py +60 -0
- basa_evals/tokenizers/mock.py +91 -0
- basa_evals/tokenizers/registry.py +94 -0
- basa_evals/tokenizers/runner.py +82 -0
- basa_evals/tokenizers/tiktoken_adapter.py +63 -0
- basa_evals-0.1.0.dist-info/METADATA +193 -0
- basa_evals-0.1.0.dist-info/RECORD +58 -0
- basa_evals-0.1.0.dist-info/WHEEL +4 -0
- basa_evals-0.1.0.dist-info/entry_points.txt +2 -0
- basa_evals-0.1.0.dist-info/licenses/LICENSE +21 -0
basa_evals/__init__.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""SUT adapters module."""
|
|
2
|
+
|
|
3
|
+
from basa_evals.adapters.basa import BasaAdapter
|
|
4
|
+
from basa_evals.adapters.base import BasePreprocessor
|
|
5
|
+
from basa_evals.adapters.identity import RawAdapter
|
|
6
|
+
from basa_evals.adapters.sastrawi import SastrawiAdapter
|
|
7
|
+
from basa_evals.adapters.simple_regex import SimpleRegexBaseline
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"BasePreprocessor",
|
|
11
|
+
"RawAdapter",
|
|
12
|
+
"SimpleRegexBaseline",
|
|
13
|
+
"SastrawiAdapter",
|
|
14
|
+
"BasaAdapter",
|
|
15
|
+
]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""BASA preprocessor adapter with construction-time signature checks."""
|
|
2
|
+
|
|
3
|
+
import inspect
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class BasaAdapter:
|
|
8
|
+
"""Adapter for the BASA normalization library."""
|
|
9
|
+
|
|
10
|
+
def __init__(
|
|
11
|
+
self,
|
|
12
|
+
system_id: str = "basa-default",
|
|
13
|
+
params: dict[str, Any] | None = None,
|
|
14
|
+
**kwargs: Any,
|
|
15
|
+
) -> None:
|
|
16
|
+
self._system_id = system_id
|
|
17
|
+
self._params = dict(params or {})
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
import basa # type: ignore[import-untyped]
|
|
21
|
+
|
|
22
|
+
self._basa = basa
|
|
23
|
+
except ImportError as e:
|
|
24
|
+
raise ImportError(
|
|
25
|
+
"basa is required for BasaAdapter. "
|
|
26
|
+
"Install via `pip install basa-evals[basa]` or install basa."
|
|
27
|
+
) from e
|
|
28
|
+
|
|
29
|
+
# Validate params against basa.normalize signature
|
|
30
|
+
sig = inspect.signature(basa.normalize)
|
|
31
|
+
valid_params = set(sig.parameters.keys()) - {"text"}
|
|
32
|
+
for k in self._params:
|
|
33
|
+
if k not in valid_params:
|
|
34
|
+
raise RuntimeError(
|
|
35
|
+
f"basa.normalize received unsupported parameter '{k}'. "
|
|
36
|
+
f"Supported parameters are: {sorted(valid_params)}"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def system_id(self) -> str:
|
|
41
|
+
return self._system_id
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def version(self) -> str | None:
|
|
45
|
+
return getattr(self._basa, "__version__", None)
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def params(self) -> dict[str, Any]:
|
|
49
|
+
return self._params
|
|
50
|
+
|
|
51
|
+
def process(self, text: str) -> str:
|
|
52
|
+
res = self._basa.normalize(text, **self._params)
|
|
53
|
+
if isinstance(res, list):
|
|
54
|
+
return str(res[0]) if res else ""
|
|
55
|
+
return str(res)
|
|
56
|
+
|
|
57
|
+
def process_batch(self, texts: list[str]) -> list[str]:
|
|
58
|
+
res = self._basa.normalize(texts, **self._params)
|
|
59
|
+
if isinstance(res, str):
|
|
60
|
+
return [res]
|
|
61
|
+
return list(res)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""BasePreprocessor protocol definition for Systems Under Test."""
|
|
2
|
+
|
|
3
|
+
from typing import Protocol, runtime_checkable
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@runtime_checkable
|
|
7
|
+
class BasePreprocessor(Protocol):
|
|
8
|
+
"""Protocol defining the interface for all Systems Under Test."""
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def system_id(self) -> str:
|
|
12
|
+
"""Unique identifier of the system."""
|
|
13
|
+
...
|
|
14
|
+
|
|
15
|
+
@property
|
|
16
|
+
def version(self) -> str | None:
|
|
17
|
+
"""Version of the underlying library or implementation, or None."""
|
|
18
|
+
...
|
|
19
|
+
|
|
20
|
+
def process(self, text: str) -> str:
|
|
21
|
+
"""Process a single text sample."""
|
|
22
|
+
...
|
|
23
|
+
|
|
24
|
+
def process_batch(self, texts: list[str]) -> list[str]:
|
|
25
|
+
"""Process a batch of texts, preserving exact order and length."""
|
|
26
|
+
...
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""Identity adapter acting as noise floor and no-op control."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class RawAdapter:
|
|
5
|
+
"""Identity preprocessor returning input strings unmodified."""
|
|
6
|
+
|
|
7
|
+
def __init__(self, system_id: str = "raw-identity", **kwargs: object) -> None:
|
|
8
|
+
self._system_id = system_id
|
|
9
|
+
|
|
10
|
+
@property
|
|
11
|
+
def system_id(self) -> str:
|
|
12
|
+
return self._system_id
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def version(self) -> str | None:
|
|
16
|
+
return None
|
|
17
|
+
|
|
18
|
+
def process(self, text: str) -> str:
|
|
19
|
+
return text
|
|
20
|
+
|
|
21
|
+
def process_batch(self, texts: list[str]) -> list[str]:
|
|
22
|
+
return list(texts)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Sastrawi morphological stemmer adapter."""
|
|
2
|
+
|
|
3
|
+
import importlib.metadata
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class SastrawiAdapter:
|
|
8
|
+
"""Stemmer baseline using PySastrawi.
|
|
9
|
+
|
|
10
|
+
Morphological stemmer, not a normalizer — included as a destructive-edit
|
|
11
|
+
stress baseline, not as a competitive normalizer.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
def __init__(self, system_id: str = "sastrawi", **kwargs: Any) -> None:
|
|
15
|
+
self._system_id = system_id
|
|
16
|
+
try:
|
|
17
|
+
from Sastrawi.Stemmer.StemmerFactory import ( # type: ignore[import-untyped]
|
|
18
|
+
StemmerFactory,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
self._stemmer = StemmerFactory().create_stemmer()
|
|
22
|
+
except ImportError as e:
|
|
23
|
+
raise ImportError(
|
|
24
|
+
"PySastrawi is required for SastrawiAdapter. "
|
|
25
|
+
"Install via `pip install basa-evals[sastrawi]` or `pip install PySastrawi`."
|
|
26
|
+
) from e
|
|
27
|
+
|
|
28
|
+
@property
|
|
29
|
+
def system_id(self) -> str:
|
|
30
|
+
return self._system_id
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def version(self) -> str | None:
|
|
34
|
+
try:
|
|
35
|
+
return importlib.metadata.version("PySastrawi")
|
|
36
|
+
except Exception:
|
|
37
|
+
return None
|
|
38
|
+
|
|
39
|
+
def process(self, text: str) -> str:
|
|
40
|
+
return str(self._stemmer.stem(text))
|
|
41
|
+
|
|
42
|
+
def process_batch(self, texts: list[str]) -> list[str]:
|
|
43
|
+
return [self.process(t) for t in texts]
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Simple regex and dictionary ablation baseline adapter."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class SimpleRegexBaseline:
|
|
9
|
+
"""Ablation baseline applying run collapse (>=3) and 50-entry dictionary lookup."""
|
|
10
|
+
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
system_id: str = "simple-regex",
|
|
14
|
+
dict_path: str | Path | None = None,
|
|
15
|
+
**kwargs: object,
|
|
16
|
+
) -> None:
|
|
17
|
+
self._system_id = system_id
|
|
18
|
+
if dict_path is None:
|
|
19
|
+
dict_path = (
|
|
20
|
+
Path(__file__).parent.parent / "data" / "baselines" / "simple_regex_dict_v1.json"
|
|
21
|
+
)
|
|
22
|
+
with open(dict_path, encoding="utf-8") as f:
|
|
23
|
+
data = json.load(f)
|
|
24
|
+
self.mappings: dict[str, str] = data.get("mappings", {})
|
|
25
|
+
|
|
26
|
+
# Precompile word boundary regex for fast dictionary replacement
|
|
27
|
+
self._pattern: re.Pattern[str] | None
|
|
28
|
+
if self.mappings:
|
|
29
|
+
escaped = [re.escape(k) for k in self.mappings.keys()]
|
|
30
|
+
self._pattern = re.compile(r"\b(" + "|".join(escaped) + r")\b")
|
|
31
|
+
else:
|
|
32
|
+
self._pattern = None
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def system_id(self) -> str:
|
|
36
|
+
return self._system_id
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def version(self) -> str | None:
|
|
40
|
+
return "1.0.0"
|
|
41
|
+
|
|
42
|
+
def process(self, text: str) -> str:
|
|
43
|
+
# 1. Lowercase
|
|
44
|
+
t = text.lower()
|
|
45
|
+
# 2. Collapse runs of >= 3 identical characters to 1
|
|
46
|
+
t = re.sub(r"(.)\1{2,}", r"\1", t)
|
|
47
|
+
# 3. Apply dictionary replacement
|
|
48
|
+
if self._pattern:
|
|
49
|
+
t = self._pattern.sub(lambda m: self.mappings[m.group(0)], t)
|
|
50
|
+
return t
|
|
51
|
+
|
|
52
|
+
def process_batch(self, texts: list[str]) -> list[str]:
|
|
53
|
+
return [self.process(t) for t in texts]
|