basa-evals 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. basa_evals/__init__.py +3 -0
  2. basa_evals/adapters/__init__.py +15 -0
  3. basa_evals/adapters/basa.py +61 -0
  4. basa_evals/adapters/base.py +26 -0
  5. basa_evals/adapters/identity.py +22 -0
  6. basa_evals/adapters/sastrawi.py +43 -0
  7. basa_evals/adapters/simple_regex.py +53 -0
  8. basa_evals/cli.py +563 -0
  9. basa_evals/configs/suites/downstream_smsa.yaml +16 -0
  10. basa_evals/configs/suites/pilot_210.yaml +33 -0
  11. basa_evals/configs/suites/tokenizer_pilot.yaml +27 -0
  12. basa_evals/core/__init__.py +1 -0
  13. basa_evals/core/alignment.py +227 -0
  14. basa_evals/core/gate.py +168 -0
  15. basa_evals/core/loader.py +109 -0
  16. basa_evals/core/provenance.py +36 -0
  17. basa_evals/core/registry.py +42 -0
  18. basa_evals/core/runner.py +188 -0
  19. basa_evals/core/schema.py +388 -0
  20. basa_evals/core/tokenizer.py +91 -0
  21. basa_evals/data/baselines/simple_regex_dict_v1.json +61 -0
  22. basa_evals/data/challenge/mixed_challenge_v1.jsonl +30 -0
  23. basa_evals/data/challenge/negative_controls_code_switching_v1.jsonl +30 -0
  24. basa_evals/data/challenge/negative_controls_kbbi_v1.jsonl +30 -0
  25. basa_evals/data/downstream/smsa_test_v1.jsonl +150 -0
  26. basa_evals/data/gold/abbreviation_phonetic_v1.jsonl +30 -0
  27. basa_evals/data/gold/repetition_reduction_v1.jsonl +30 -0
  28. basa_evals/data/gold/slang_lexical_v1.jsonl +30 -0
  29. basa_evals/data/gold/typo_nonword_v1.jsonl +30 -0
  30. basa_evals/downstream/__init__.py +9 -0
  31. basa_evals/downstream/base.py +18 -0
  32. basa_evals/downstream/hf.py +59 -0
  33. basa_evals/downstream/mock.py +72 -0
  34. basa_evals/downstream/registry.py +57 -0
  35. basa_evals/downstream/runner.py +93 -0
  36. basa_evals/metrics/__init__.py +29 -0
  37. basa_evals/metrics/aggregate.py +168 -0
  38. basa_evals/metrics/downstream.py +167 -0
  39. basa_evals/metrics/linguistic.py +65 -0
  40. basa_evals/metrics/preservation.py +31 -0
  41. basa_evals/metrics/tokenization.py +173 -0
  42. basa_evals/py.typed +1 -0
  43. basa_evals/reporters/__init__.py +24 -0
  44. basa_evals/reporters/console.py +331 -0
  45. basa_evals/reporters/exporter.py +75 -0
  46. basa_evals/reporters/markdown.py +223 -0
  47. basa_evals/tokenizers/__init__.py +13 -0
  48. basa_evals/tokenizers/base.py +31 -0
  49. basa_evals/tokenizers/hf_adapter.py +60 -0
  50. basa_evals/tokenizers/mock.py +91 -0
  51. basa_evals/tokenizers/registry.py +94 -0
  52. basa_evals/tokenizers/runner.py +82 -0
  53. basa_evals/tokenizers/tiktoken_adapter.py +63 -0
  54. basa_evals-0.1.0.dist-info/METADATA +193 -0
  55. basa_evals-0.1.0.dist-info/RECORD +58 -0
  56. basa_evals-0.1.0.dist-info/WHEEL +4 -0
  57. basa_evals-0.1.0.dist-info/entry_points.txt +2 -0
  58. basa_evals-0.1.0.dist-info/licenses/LICENSE +21 -0
basa_evals/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """BASA Evals: Indonesian NLP Preprocessing & Normalization Benchmark Suite."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,15 @@
1
+ """SUT adapters module."""
2
+
3
+ from basa_evals.adapters.basa import BasaAdapter
4
+ from basa_evals.adapters.base import BasePreprocessor
5
+ from basa_evals.adapters.identity import RawAdapter
6
+ from basa_evals.adapters.sastrawi import SastrawiAdapter
7
+ from basa_evals.adapters.simple_regex import SimpleRegexBaseline
8
+
9
+ __all__ = [
10
+ "BasePreprocessor",
11
+ "RawAdapter",
12
+ "SimpleRegexBaseline",
13
+ "SastrawiAdapter",
14
+ "BasaAdapter",
15
+ ]
@@ -0,0 +1,61 @@
1
+ """BASA preprocessor adapter with construction-time signature checks."""
2
+
3
+ import inspect
4
+ from typing import Any
5
+
6
+
7
+ class BasaAdapter:
8
+ """Adapter for the BASA normalization library."""
9
+
10
+ def __init__(
11
+ self,
12
+ system_id: str = "basa-default",
13
+ params: dict[str, Any] | None = None,
14
+ **kwargs: Any,
15
+ ) -> None:
16
+ self._system_id = system_id
17
+ self._params = dict(params or {})
18
+
19
+ try:
20
+ import basa # type: ignore[import-untyped]
21
+
22
+ self._basa = basa
23
+ except ImportError as e:
24
+ raise ImportError(
25
+ "basa is required for BasaAdapter. "
26
+ "Install via `pip install basa-evals[basa]` or install basa."
27
+ ) from e
28
+
29
+ # Validate params against basa.normalize signature
30
+ sig = inspect.signature(basa.normalize)
31
+ valid_params = set(sig.parameters.keys()) - {"text"}
32
+ for k in self._params:
33
+ if k not in valid_params:
34
+ raise RuntimeError(
35
+ f"basa.normalize received unsupported parameter '{k}'. "
36
+ f"Supported parameters are: {sorted(valid_params)}"
37
+ )
38
+
39
+ @property
40
+ def system_id(self) -> str:
41
+ return self._system_id
42
+
43
+ @property
44
+ def version(self) -> str | None:
45
+ return getattr(self._basa, "__version__", None)
46
+
47
+ @property
48
+ def params(self) -> dict[str, Any]:
49
+ return self._params
50
+
51
+ def process(self, text: str) -> str:
52
+ res = self._basa.normalize(text, **self._params)
53
+ if isinstance(res, list):
54
+ return str(res[0]) if res else ""
55
+ return str(res)
56
+
57
+ def process_batch(self, texts: list[str]) -> list[str]:
58
+ res = self._basa.normalize(texts, **self._params)
59
+ if isinstance(res, str):
60
+ return [res]
61
+ return list(res)
@@ -0,0 +1,26 @@
1
+ """BasePreprocessor protocol definition for Systems Under Test."""
2
+
3
+ from typing import Protocol, runtime_checkable
4
+
5
+
6
+ @runtime_checkable
7
+ class BasePreprocessor(Protocol):
8
+ """Protocol defining the interface for all Systems Under Test."""
9
+
10
+ @property
11
+ def system_id(self) -> str:
12
+ """Unique identifier of the system."""
13
+ ...
14
+
15
+ @property
16
+ def version(self) -> str | None:
17
+ """Version of the underlying library or implementation, or None."""
18
+ ...
19
+
20
+ def process(self, text: str) -> str:
21
+ """Process a single text sample."""
22
+ ...
23
+
24
+ def process_batch(self, texts: list[str]) -> list[str]:
25
+ """Process a batch of texts, preserving exact order and length."""
26
+ ...
@@ -0,0 +1,22 @@
1
+ """Identity adapter acting as noise floor and no-op control."""
2
+
3
+
4
+ class RawAdapter:
5
+ """Identity preprocessor returning input strings unmodified."""
6
+
7
+ def __init__(self, system_id: str = "raw-identity", **kwargs: object) -> None:
8
+ self._system_id = system_id
9
+
10
+ @property
11
+ def system_id(self) -> str:
12
+ return self._system_id
13
+
14
+ @property
15
+ def version(self) -> str | None:
16
+ return None
17
+
18
+ def process(self, text: str) -> str:
19
+ return text
20
+
21
+ def process_batch(self, texts: list[str]) -> list[str]:
22
+ return list(texts)
@@ -0,0 +1,43 @@
1
+ """Sastrawi morphological stemmer adapter."""
2
+
3
+ import importlib.metadata
4
+ from typing import Any
5
+
6
+
7
+ class SastrawiAdapter:
8
+ """Stemmer baseline using PySastrawi.
9
+
10
+ Morphological stemmer, not a normalizer — included as a destructive-edit
11
+ stress baseline, not as a competitive normalizer.
12
+ """
13
+
14
+ def __init__(self, system_id: str = "sastrawi", **kwargs: Any) -> None:
15
+ self._system_id = system_id
16
+ try:
17
+ from Sastrawi.Stemmer.StemmerFactory import ( # type: ignore[import-untyped]
18
+ StemmerFactory,
19
+ )
20
+
21
+ self._stemmer = StemmerFactory().create_stemmer()
22
+ except ImportError as e:
23
+ raise ImportError(
24
+ "PySastrawi is required for SastrawiAdapter. "
25
+ "Install via `pip install basa-evals[sastrawi]` or `pip install PySastrawi`."
26
+ ) from e
27
+
28
+ @property
29
+ def system_id(self) -> str:
30
+ return self._system_id
31
+
32
+ @property
33
+ def version(self) -> str | None:
34
+ try:
35
+ return importlib.metadata.version("PySastrawi")
36
+ except Exception:
37
+ return None
38
+
39
+ def process(self, text: str) -> str:
40
+ return str(self._stemmer.stem(text))
41
+
42
+ def process_batch(self, texts: list[str]) -> list[str]:
43
+ return [self.process(t) for t in texts]
@@ -0,0 +1,53 @@
1
+ """Simple regex and dictionary ablation baseline adapter."""
2
+
3
+ import json
4
+ import re
5
+ from pathlib import Path
6
+
7
+
8
+ class SimpleRegexBaseline:
9
+ """Ablation baseline applying run collapse (>=3) and 50-entry dictionary lookup."""
10
+
11
+ def __init__(
12
+ self,
13
+ system_id: str = "simple-regex",
14
+ dict_path: str | Path | None = None,
15
+ **kwargs: object,
16
+ ) -> None:
17
+ self._system_id = system_id
18
+ if dict_path is None:
19
+ dict_path = (
20
+ Path(__file__).parent.parent / "data" / "baselines" / "simple_regex_dict_v1.json"
21
+ )
22
+ with open(dict_path, encoding="utf-8") as f:
23
+ data = json.load(f)
24
+ self.mappings: dict[str, str] = data.get("mappings", {})
25
+
26
+ # Precompile word boundary regex for fast dictionary replacement
27
+ self._pattern: re.Pattern[str] | None
28
+ if self.mappings:
29
+ escaped = [re.escape(k) for k in self.mappings.keys()]
30
+ self._pattern = re.compile(r"\b(" + "|".join(escaped) + r")\b")
31
+ else:
32
+ self._pattern = None
33
+
34
+ @property
35
+ def system_id(self) -> str:
36
+ return self._system_id
37
+
38
+ @property
39
+ def version(self) -> str | None:
40
+ return "1.0.0"
41
+
42
+ def process(self, text: str) -> str:
43
+ # 1. Lowercase
44
+ t = text.lower()
45
+ # 2. Collapse runs of >= 3 identical characters to 1
46
+ t = re.sub(r"(.)\1{2,}", r"\1", t)
47
+ # 3. Apply dictionary replacement
48
+ if self._pattern:
49
+ t = self._pattern.sub(lambda m: self.mappings[m.group(0)], t)
50
+ return t
51
+
52
+ def process_batch(self, texts: list[str]) -> list[str]:
53
+ return [self.process(t) for t in texts]