basa-evals 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. basa_evals-0.1.0/.gitignore +24 -0
  2. basa_evals-0.1.0/LICENSE +21 -0
  3. basa_evals-0.1.0/PKG-INFO +193 -0
  4. basa_evals-0.1.0/README.md +157 -0
  5. basa_evals-0.1.0/configs/suites/downstream_smsa.yaml +16 -0
  6. basa_evals-0.1.0/configs/suites/pilot_210.yaml +33 -0
  7. basa_evals-0.1.0/configs/suites/tokenizer_pilot.yaml +27 -0
  8. basa_evals-0.1.0/data/challenge/mixed_challenge_v1.jsonl +30 -0
  9. basa_evals-0.1.0/data/challenge/negative_controls_code_switching_v1.jsonl +30 -0
  10. basa_evals-0.1.0/data/challenge/negative_controls_kbbi_v1.jsonl +30 -0
  11. basa_evals-0.1.0/data/downstream/smsa_test_v1.jsonl +150 -0
  12. basa_evals-0.1.0/data/gold/abbreviation_phonetic_v1.jsonl +30 -0
  13. basa_evals-0.1.0/data/gold/repetition_reduction_v1.jsonl +30 -0
  14. basa_evals-0.1.0/data/gold/slang_lexical_v1.jsonl +30 -0
  15. basa_evals-0.1.0/data/gold/typo_nonword_v1.jsonl +30 -0
  16. basa_evals-0.1.0/docs/annotation_guidelines.md +216 -0
  17. basa_evals-0.1.0/docs/findings/m1-baseline.md +105 -0
  18. basa_evals-0.1.0/docs/findings/m2-tokenizer-benchmark.md +73 -0
  19. basa_evals-0.1.0/docs/findings/m3-downstream-benchmark.md +84 -0
  20. basa_evals-0.1.0/docs/superpowers/plans/2026-10-22-basa-evals-milestone-1.md +859 -0
  21. basa_evals-0.1.0/docs/superpowers/plans/2026-10-23-basa-evals-milestone-2.md +427 -0
  22. basa_evals-0.1.0/docs/superpowers/plans/2026-10-23-basa-evals-milestone-3-smsa.md +647 -0
  23. basa_evals-0.1.0/docs/superpowers/specs/2026-10-22-basa-evals-milestone-1-design.md +842 -0
  24. basa_evals-0.1.0/docs/superpowers/specs/2026-10-23-basa-evals-milestone-2-design.md +274 -0
  25. basa_evals-0.1.0/docs/superpowers/specs/2026-10-23-basa-evals-milestone-3-design.md +269 -0
  26. basa_evals-0.1.0/pyproject.toml +88 -0
  27. basa_evals-0.1.0/src/basa_evals/__init__.py +3 -0
  28. basa_evals-0.1.0/src/basa_evals/adapters/__init__.py +15 -0
  29. basa_evals-0.1.0/src/basa_evals/adapters/basa.py +61 -0
  30. basa_evals-0.1.0/src/basa_evals/adapters/base.py +26 -0
  31. basa_evals-0.1.0/src/basa_evals/adapters/identity.py +22 -0
  32. basa_evals-0.1.0/src/basa_evals/adapters/sastrawi.py +43 -0
  33. basa_evals-0.1.0/src/basa_evals/adapters/simple_regex.py +53 -0
  34. basa_evals-0.1.0/src/basa_evals/cli.py +563 -0
  35. basa_evals-0.1.0/src/basa_evals/configs/suites/downstream_smsa.yaml +16 -0
  36. basa_evals-0.1.0/src/basa_evals/configs/suites/pilot_210.yaml +33 -0
  37. basa_evals-0.1.0/src/basa_evals/configs/suites/tokenizer_pilot.yaml +27 -0
  38. basa_evals-0.1.0/src/basa_evals/core/__init__.py +1 -0
  39. basa_evals-0.1.0/src/basa_evals/core/alignment.py +227 -0
  40. basa_evals-0.1.0/src/basa_evals/core/gate.py +168 -0
  41. basa_evals-0.1.0/src/basa_evals/core/loader.py +109 -0
  42. basa_evals-0.1.0/src/basa_evals/core/provenance.py +36 -0
  43. basa_evals-0.1.0/src/basa_evals/core/registry.py +42 -0
  44. basa_evals-0.1.0/src/basa_evals/core/runner.py +188 -0
  45. basa_evals-0.1.0/src/basa_evals/core/schema.py +388 -0
  46. basa_evals-0.1.0/src/basa_evals/core/tokenizer.py +91 -0
  47. basa_evals-0.1.0/src/basa_evals/data/baselines/simple_regex_dict_v1.json +61 -0
  48. basa_evals-0.1.0/src/basa_evals/data/challenge/mixed_challenge_v1.jsonl +30 -0
  49. basa_evals-0.1.0/src/basa_evals/data/challenge/negative_controls_code_switching_v1.jsonl +30 -0
  50. basa_evals-0.1.0/src/basa_evals/data/challenge/negative_controls_kbbi_v1.jsonl +30 -0
  51. basa_evals-0.1.0/src/basa_evals/data/downstream/smsa_test_v1.jsonl +150 -0
  52. basa_evals-0.1.0/src/basa_evals/data/gold/abbreviation_phonetic_v1.jsonl +30 -0
  53. basa_evals-0.1.0/src/basa_evals/data/gold/repetition_reduction_v1.jsonl +30 -0
  54. basa_evals-0.1.0/src/basa_evals/data/gold/slang_lexical_v1.jsonl +30 -0
  55. basa_evals-0.1.0/src/basa_evals/data/gold/typo_nonword_v1.jsonl +30 -0
  56. basa_evals-0.1.0/src/basa_evals/downstream/__init__.py +9 -0
  57. basa_evals-0.1.0/src/basa_evals/downstream/base.py +18 -0
  58. basa_evals-0.1.0/src/basa_evals/downstream/hf.py +59 -0
  59. basa_evals-0.1.0/src/basa_evals/downstream/mock.py +72 -0
  60. basa_evals-0.1.0/src/basa_evals/downstream/registry.py +57 -0
  61. basa_evals-0.1.0/src/basa_evals/downstream/runner.py +93 -0
  62. basa_evals-0.1.0/src/basa_evals/metrics/__init__.py +29 -0
  63. basa_evals-0.1.0/src/basa_evals/metrics/aggregate.py +168 -0
  64. basa_evals-0.1.0/src/basa_evals/metrics/downstream.py +167 -0
  65. basa_evals-0.1.0/src/basa_evals/metrics/linguistic.py +65 -0
  66. basa_evals-0.1.0/src/basa_evals/metrics/preservation.py +31 -0
  67. basa_evals-0.1.0/src/basa_evals/metrics/tokenization.py +173 -0
  68. basa_evals-0.1.0/src/basa_evals/py.typed +1 -0
  69. basa_evals-0.1.0/src/basa_evals/reporters/__init__.py +24 -0
  70. basa_evals-0.1.0/src/basa_evals/reporters/console.py +331 -0
  71. basa_evals-0.1.0/src/basa_evals/reporters/exporter.py +75 -0
  72. basa_evals-0.1.0/src/basa_evals/reporters/markdown.py +223 -0
  73. basa_evals-0.1.0/src/basa_evals/tokenizers/__init__.py +13 -0
  74. basa_evals-0.1.0/src/basa_evals/tokenizers/base.py +31 -0
  75. basa_evals-0.1.0/src/basa_evals/tokenizers/hf_adapter.py +60 -0
  76. basa_evals-0.1.0/src/basa_evals/tokenizers/mock.py +91 -0
  77. basa_evals-0.1.0/src/basa_evals/tokenizers/registry.py +94 -0
  78. basa_evals-0.1.0/src/basa_evals/tokenizers/runner.py +82 -0
  79. basa_evals-0.1.0/src/basa_evals/tokenizers/tiktoken_adapter.py +63 -0
@@ -0,0 +1,24 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # Distribution / packaging
7
+ dist/
8
+ build/
9
+ *.egg-info/
10
+
11
+ # Virtual environments
12
+ .venv/
13
+ env/
14
+ venv/
15
+
16
+ # Testing and typing caches
17
+ .pytest_cache/
18
+ .mypy_cache/
19
+ .ruff_cache/
20
+ .hypothesis/
21
+
22
+ # Benchmark execution outputs
23
+ runs/
24
+ artifacts/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Muanai
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,193 @@
1
+ Metadata-Version: 2.5
2
+ Name: basa-evals
3
+ Version: 0.1.0
4
+ Summary: Independent evaluation framework, benchmark suite, and dataset pipeline for Indonesian NLP text normalization and preprocessing.
5
+ Author-email: Muanai <muanai@example.com>
6
+ License: MIT
7
+ License-File: LICENSE
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: Intended Audience :: Science/Research
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3.12
13
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
14
+ Classifier: Topic :: Text Processing :: Linguistic
15
+ Requires-Python: >=3.12
16
+ Requires-Dist: pydantic>=2.7.0
17
+ Requires-Dist: pyyaml>=6.0.1
18
+ Requires-Dist: rich>=13.7.0
19
+ Requires-Dist: typer>=0.12.0
20
+ Provides-Extra: basa
21
+ Requires-Dist: basa; extra == 'basa'
22
+ Provides-Extra: dev
23
+ Requires-Dist: hypothesis>=6.100.0; extra == 'dev'
24
+ Requires-Dist: mypy>=1.10.0; extra == 'dev'
25
+ Requires-Dist: pytest>=8.0.0; extra == 'dev'
26
+ Requires-Dist: ruff>=0.4.0; extra == 'dev'
27
+ Provides-Extra: downstream
28
+ Requires-Dist: torch>=2.0.0; extra == 'downstream'
29
+ Requires-Dist: transformers>=4.40.0; extra == 'downstream'
30
+ Provides-Extra: sastrawi
31
+ Requires-Dist: pysastrawi>=1.2.0; extra == 'sastrawi'
32
+ Provides-Extra: tokenizers
33
+ Requires-Dist: tiktoken>=0.7.0; extra == 'tokenizers'
34
+ Requires-Dist: transformers>=4.40.0; extra == 'tokenizers'
35
+ Description-Content-Type: text/markdown
36
+
37
+ # BASA Evals (`basa-evals`)
38
+
39
+ [![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/)
40
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
41
+ [![Data License: CC-BY-4.0](https://img.shields.io/badge/Data%20License-CC--BY--4.0-lightgrey.svg)](https://creativecommons.org/licenses/by/4.0/)
42
+
43
+ **BASA Evals** is an independent, neutral evaluation framework, benchmark suite, and dataset pipeline designed to systematically measure text normalization and preprocessing for Indonesian NLP (standard Indonesian, Bahasa Gaul slang, character repetition, non-word typos, and code-switching).
44
+
45
+ It treats every normalizer — including BASA — as an external **System Under Test (SUT)** behind a uniform protocol (`BasePreprocessor`), and measures normalization quality across three rigorous evaluation tiers: **Intrinsic Linguistic Fidelity**, **Multi-Tokenizer Efficiency**, and **Applied Downstream Task Impact**.
46
+
47
+ ---
48
+
49
+ ## 1. Core Mission & Philosophy
50
+
51
+ The framework answers one question without favour: **does a given normalizer improve informal Indonesian text without destroying clean text or hurting downstream models?**
52
+
53
+ - **Preservation First:** Under-normalization (slang left untouched) costs tokenizer efficiency. Over-normalization (clean KBBI text, brand names, technical terms, and code-switched jargon mutated into something else) corrupts meaning silently upstream of LLM and RAG pipelines.
54
+ - **Mandatory Negative Controls:** Every intrinsic benchmark suite must contain at least **20% negative-control samples** (`raw_input == expected_output`).
55
+ - **Strict Release Gatekeeper:** Candidate releases must achieve $\text{ONER} = 0$ (zero errors) on pilot slices ($N < 500$ clean tokens) and $\text{ONER} < 0.5\%$ on scaled benchmark suites.
56
+ - **Three-Tier Evaluation:** Normalization is evaluated intrinsically (edit distance & preservation), structurally (subword tokenizer fragmentation), and operationally (classification accuracy and $F_1$ deltas).
57
+
58
+ ---
59
+
60
+ ## 2. Three-Tier Benchmark Suites
61
+
62
+ ### Tier 1: Intrinsic Linguistic Precision (`pilot_210`)
63
+ Measures exact transformation correctness and negative-control preservation across 210 curated rows:
64
+
65
+ | Slice | Rows | Category | Target Phenomenon | Gate Criteria |
66
+ |---|---:|---|---|---|
67
+ | `negative_controls_kbbi` | 30 | Negative Control | Formal Indonesian, URLs, numbers, entities | $\text{ONER} = 0$, $\text{Entity} = 100\%$ |
68
+ | `negative_controls_code_switching` | 30 | Negative Control | Indonesian with preserved English IT terms | $\text{ONER} = 0$, $\text{Entity} = 100\%$ |
69
+ | `repetition_reduction` | 30 | Gold Slice | Character runs $\ge 3$ on standard words | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
70
+ | `slang_lexical` | 30 | Gold Slice | Colloquial slang dictionary mapping | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
71
+ | `abbreviation_phonetic` | 30 | Gold Slice | Consonant skeletons and omitted vowels | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
72
+ | `typo_nonword` | 30 | Gold Slice | Single-edit non-word typos | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
73
+ | `mixed_challenge` | 30 | Exploratory | Composed phenomena ($\ge 2$ operations) | *Ungated diagnostics* |
74
+
75
+ Negative control ratio: **60 / 180 = 33.3%** of the gated population, **60 / 210 = 28.6%** of the total.
76
+
77
+ ### Tier 2: Multi-Tokenizer Efficiency (`tokenizer_pilot`)
78
+ Measures whether normalization reduces prompt token counts and mitigates context fragmentation across leading foundation models:
79
+ - **Evaluated Tokenizers:** OpenAI GPT-4o (`o200k_base`), Meta LLaMA-3 (BPE 128k), IndoBERT (WordPiece 32k), Gemma-2 (SentencePiece), and lightweight offline mock subword tokenizers.
80
+
81
+ ### Tier 3: Applied Downstream Tasks (`downstream_smsa`)
82
+ Measures downstream task accuracy deltas on real-world Indonesian customer review sentiment:
83
+ - **Corpus:** 150 balanced IndoNLU SmSA test samples (50 positive, 50 neutral, 50 negative).
84
+ - **Classifiers:** Built-in offline `MockSentimentClassifier` (zero external dependencies) and fine-tuned `HuggingFaceSentimentClassifier` (`wirasana/indobert-sentiment-smsa`).
85
+
86
+ ---
87
+
88
+ ## 3. Metrics Formulation
89
+
90
+ ### Intrinsic Metrics (Tier 1)
91
+ - **Exact Match (Dual Metric):**
92
+ - `EM_exact`: String equality after NFC normalization and whitespace stripping.
93
+ - `EM_casefold`: Case-insensitive exact match after NFC normalization.
94
+ - **Character Error Rate (CER):** $\sum \text{Lev}_{\text{chars}}(\text{pred}, \text{gold}) / \sum \max(1, \text{len}(\text{gold}))$.
95
+ - **Word Error Rate (WER):** $\sum \text{Lev}_{\text{words}}(\text{pred}, \text{gold}) / \sum \max(1, |\text{gold word tokens}|)$.
96
+ - **Over-Normalization Error Rate (ONER):** $\sum \text{ON Errors} / \sum \text{Clean Tokens}$.
97
+ - **Under-Normalization Error Rate (UNER):** $\sum \text{UN Errors} / \sum \text{Noisy Tokens}$.
98
+ - **Entity Preservation Rate:** $\sum \text{Entities Preserved} / \sum \text{Entities Expected}$.
99
+
100
+ ### Tokenizer Efficiency Metrics (Tier 2)
101
+ - **Token Compression Ratio:** $\text{Raw Tokens} / \max(1, \text{Pred Tokens})$.
102
+ - **Net Savings Rate:** $(\text{Raw Tokens} - \text{Pred Tokens}) / \text{Raw Tokens}$.
103
+ - **Subword Fragmentation Index (SFI):** Average subwords per whitespace-delimited word ($\text{SFI} = T / W$). Lower SFI indicates reduced fragmentation.
104
+
105
+ ### Downstream Task Metrics (Tier 3)
106
+ - **Accuracy & Macro $F_1$ Deltas:** $\Delta\text{Acc} = \text{Acc}_{\text{norm}} - \text{Acc}_{\text{raw}}$, $\Delta F_1 = F_{1,\text{norm}} - F_{1,\text{raw}}$.
107
+ - **Sample Outcome Partitioning:** Every sample is partitioned into **Win**, **Loss**, or **Neutral** under normalization:
108
+ - $\text{Win}$: Raw incorrect, normalized correct.
109
+ - $\text{Loss}$: Raw correct, normalized incorrect.
110
+ - $\text{Neutral}$: Both correct or both incorrect.
111
+ - **Net Win Rate:** $(\text{Wins} - \text{Losses}) / N_{\text{total}}$.
112
+ - **Conservation Invariant:** $\text{Wins} + \text{Losses} + \text{Neutrals} = N_{\text{total}}$ holds strictly across all systems, with zero delta on `raw-identity`.
113
+
114
+ ---
115
+
116
+ ## 4. Quickstart & Installation
117
+
118
+ ```bash
119
+ # Basic installation (pure Python, zero heavy dependencies)
120
+ pip install basa-evals
121
+
122
+ # With subword tokenizers (tiktoken, transformers)
123
+ pip install "basa-evals[tokenizers]"
124
+
125
+ # With downstream transformer classifiers (torch, transformers)
126
+ pip install "basa-evals[downstream]"
127
+
128
+ # Full installation including SUT baselines (basa, PySastrawi)
129
+ pip install "basa-evals[basa,sastrawi,tokenizers,downstream,dev]"
130
+ ```
131
+
132
+ ### CLI Commands
133
+
134
+ #### Tier 1: Intrinsic Linguistic Evaluation
135
+ ```bash
136
+ # Run the pilot suite across all registered systems
137
+ basa-eval run --suite pilot_210
138
+
139
+ # Run evaluation on a specific system
140
+ basa-eval run --suite pilot_210 --system basa-default
141
+
142
+ # Run in advisory mode (record results without failing on gate exit code)
143
+ basa-eval run --suite pilot_210 --no-gate
144
+ ```
145
+
146
+ #### Tier 2: Multi-Tokenizer Efficiency Benchmark
147
+ ```bash
148
+ # Run tokenizer benchmark across GPT-4o and LLaMA-3
149
+ basa-eval tokenize --suite tokenizer_pilot --tokenizers gpt-4o,llama-3
150
+
151
+ # Run with offline mock tokenizer (zero dependencies)
152
+ basa-eval tokenize --suite tokenizer_pilot --tokenizers mock-subword
153
+ ```
154
+
155
+ #### Tier 3: Applied Downstream Task Benchmark
156
+ ```bash
157
+ # Run sentiment classification benchmark on IndoNLU SmSA test split
158
+ basa-eval downstream --suite downstream_smsa --classifier mock-sentiment
159
+
160
+ # Export JSON and Markdown benchmark reports
161
+ basa-eval downstream --suite downstream_smsa --classifier mock-sentiment --output-dir reports/
162
+ ```
163
+
164
+ #### Discovery & Inspection Utilities
165
+ ```bash
166
+ # List available suites, systems, tokenizers, and classifiers
167
+ basa-eval list-suites
168
+ basa-eval list-systems
169
+ basa-eval list-tokenizers
170
+ basa-eval list-classifiers
171
+
172
+ # Validate dataset integrity across all slices
173
+ basa-eval validate-dataset
174
+ ```
175
+
176
+ ---
177
+
178
+ ## 5. Explicit Non-Goals & Known Limitations
179
+
180
+ | Limitation | Statement |
181
+ |---|---|
182
+ | **Single-phenomenon scope** | In Tier 1, the six gated slices each target one isolated phenomenon. Multi-phenomenon interactions are measured in the exploratory `mixed_challenge` slice. |
183
+ | **Case invariance** | ONER and entity preservation compare case-folded tokens. Legitimate capitalization changes are monitored via the `EM exact` diagnostic column. |
184
+ | **Punctuation invariance** | Punctuation tokens are ineligible for ONER. Punctuation edits appear in CER/WER only and do not fail the preservation gate. |
185
+ | **Statistical inference** | Benchmark slices in the pilot are curated at 30 rows per phenomenon (150 rows in SmSA). Differences below ~10 percentage points on small slices should be interpreted directionally. |
186
+
187
+ ---
188
+
189
+ ## 6. License & Data Governance
190
+
191
+ - **Software Code:** [MIT License](LICENSE)
192
+ - **Evaluation Slices:** [Creative Commons Attribution 4.0 International (CC-BY-4.0)](https://creativecommons.org/licenses/by/4.0/)
193
+ - All benchmark slices are manually curated or synthetically constructed under explicit provenance. No raw, unconsented social media scrapes are used.
@@ -0,0 +1,157 @@
1
+ # BASA Evals (`basa-evals`)
2
+
3
+ [![Python 3.12+](https://img.shields.io/badge/python-3.12+-blue.svg)](https://www.python.org/downloads/)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
5
+ [![Data License: CC-BY-4.0](https://img.shields.io/badge/Data%20License-CC--BY--4.0-lightgrey.svg)](https://creativecommons.org/licenses/by/4.0/)
6
+
7
+ **BASA Evals** is an independent, neutral evaluation framework, benchmark suite, and dataset pipeline designed to systematically measure text normalization and preprocessing for Indonesian NLP (standard Indonesian, Bahasa Gaul slang, character repetition, non-word typos, and code-switching).
8
+
9
+ It treats every normalizer — including BASA — as an external **System Under Test (SUT)** behind a uniform protocol (`BasePreprocessor`), and measures normalization quality across three rigorous evaluation tiers: **Intrinsic Linguistic Fidelity**, **Multi-Tokenizer Efficiency**, and **Applied Downstream Task Impact**.
10
+
11
+ ---
12
+
13
+ ## 1. Core Mission & Philosophy
14
+
15
+ The framework answers one question without favour: **does a given normalizer improve informal Indonesian text without destroying clean text or hurting downstream models?**
16
+
17
+ - **Preservation First:** Under-normalization (slang left untouched) costs tokenizer efficiency. Over-normalization (clean KBBI text, brand names, technical terms, and code-switched jargon mutated into something else) corrupts meaning silently upstream of LLM and RAG pipelines.
18
+ - **Mandatory Negative Controls:** Every intrinsic benchmark suite must contain at least **20% negative-control samples** (`raw_input == expected_output`).
19
+ - **Strict Release Gatekeeper:** Candidate releases must achieve $\text{ONER} = 0$ (zero errors) on pilot slices ($N < 500$ clean tokens) and $\text{ONER} < 0.5\%$ on scaled benchmark suites.
20
+ - **Three-Tier Evaluation:** Normalization is evaluated intrinsically (edit distance & preservation), structurally (subword tokenizer fragmentation), and operationally (classification accuracy and $F_1$ deltas).
21
+
22
+ ---
23
+
24
+ ## 2. Three-Tier Benchmark Suites
25
+
26
+ ### Tier 1: Intrinsic Linguistic Precision (`pilot_210`)
27
+ Measures exact transformation correctness and negative-control preservation across 210 curated rows:
28
+
29
+ | Slice | Rows | Category | Target Phenomenon | Gate Criteria |
30
+ |---|---:|---|---|---|
31
+ | `negative_controls_kbbi` | 30 | Negative Control | Formal Indonesian, URLs, numbers, entities | $\text{ONER} = 0$, $\text{Entity} = 100\%$ |
32
+ | `negative_controls_code_switching` | 30 | Negative Control | Indonesian with preserved English IT terms | $\text{ONER} = 0$, $\text{Entity} = 100\%$ |
33
+ | `repetition_reduction` | 30 | Gold Slice | Character runs $\ge 3$ on standard words | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
34
+ | `slang_lexical` | 30 | Gold Slice | Colloquial slang dictionary mapping | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
35
+ | `abbreviation_phonetic` | 30 | Gold Slice | Consonant skeletons and omitted vowels | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
36
+ | `typo_nonword` | 30 | Gold Slice | Single-edit non-word typos | $\text{ONER} = 0$, $\text{UNER} \le 15\%$ |
37
+ | `mixed_challenge` | 30 | Exploratory | Composed phenomena ($\ge 2$ operations) | *Ungated diagnostics* |
38
+
39
+ Negative control ratio: **60 / 180 = 33.3%** of the gated population, **60 / 210 = 28.6%** of the total.
40
+
41
+ ### Tier 2: Multi-Tokenizer Efficiency (`tokenizer_pilot`)
42
+ Measures whether normalization reduces prompt token counts and mitigates context fragmentation across leading foundation models:
43
+ - **Evaluated Tokenizers:** OpenAI GPT-4o (`o200k_base`), Meta LLaMA-3 (BPE 128k), IndoBERT (WordPiece 32k), Gemma-2 (SentencePiece), and lightweight offline mock subword tokenizers.
44
+
45
+ ### Tier 3: Applied Downstream Tasks (`downstream_smsa`)
46
+ Measures downstream task accuracy deltas on real-world Indonesian customer review sentiment:
47
+ - **Corpus:** 150 balanced IndoNLU SmSA test samples (50 positive, 50 neutral, 50 negative).
48
+ - **Classifiers:** Built-in offline `MockSentimentClassifier` (zero external dependencies) and fine-tuned `HuggingFaceSentimentClassifier` (`wirasana/indobert-sentiment-smsa`).
49
+
50
+ ---
51
+
52
+ ## 3. Metrics Formulation
53
+
54
+ ### Intrinsic Metrics (Tier 1)
55
+ - **Exact Match (Dual Metric):**
56
+ - `EM_exact`: String equality after NFC normalization and whitespace stripping.
57
+ - `EM_casefold`: Case-insensitive exact match after NFC normalization.
58
+ - **Character Error Rate (CER):** $\sum \text{Lev}_{\text{chars}}(\text{pred}, \text{gold}) / \sum \max(1, \text{len}(\text{gold}))$.
59
+ - **Word Error Rate (WER):** $\sum \text{Lev}_{\text{words}}(\text{pred}, \text{gold}) / \sum \max(1, |\text{gold word tokens}|)$.
60
+ - **Over-Normalization Error Rate (ONER):** $\sum \text{ON Errors} / \sum \text{Clean Tokens}$.
61
+ - **Under-Normalization Error Rate (UNER):** $\sum \text{UN Errors} / \sum \text{Noisy Tokens}$.
62
+ - **Entity Preservation Rate:** $\sum \text{Entities Preserved} / \sum \text{Entities Expected}$.
63
+
64
+ ### Tokenizer Efficiency Metrics (Tier 2)
65
+ - **Token Compression Ratio:** $\text{Raw Tokens} / \max(1, \text{Pred Tokens})$.
66
+ - **Net Savings Rate:** $(\text{Raw Tokens} - \text{Pred Tokens}) / \text{Raw Tokens}$.
67
+ - **Subword Fragmentation Index (SFI):** Average subwords per whitespace-delimited word ($\text{SFI} = T / W$). Lower SFI indicates reduced fragmentation.
68
+
69
+ ### Downstream Task Metrics (Tier 3)
70
+ - **Accuracy & Macro $F_1$ Deltas:** $\Delta\text{Acc} = \text{Acc}_{\text{norm}} - \text{Acc}_{\text{raw}}$, $\Delta F_1 = F_{1,\text{norm}} - F_{1,\text{raw}}$.
71
+ - **Sample Outcome Partitioning:** Every sample is partitioned into **Win**, **Loss**, or **Neutral** under normalization:
72
+ - $\text{Win}$: Raw incorrect, normalized correct.
73
+ - $\text{Loss}$: Raw correct, normalized incorrect.
74
+ - $\text{Neutral}$: Both correct or both incorrect.
75
+ - **Net Win Rate:** $(\text{Wins} - \text{Losses}) / N_{\text{total}}$.
76
+ - **Conservation Invariant:** $\text{Wins} + \text{Losses} + \text{Neutrals} = N_{\text{total}}$ holds strictly across all systems, with zero delta on `raw-identity`.
77
+
78
+ ---
79
+
80
+ ## 4. Quickstart & Installation
81
+
82
+ ```bash
83
+ # Basic installation (pure Python, zero heavy dependencies)
84
+ pip install basa-evals
85
+
86
+ # With subword tokenizers (tiktoken, transformers)
87
+ pip install "basa-evals[tokenizers]"
88
+
89
+ # With downstream transformer classifiers (torch, transformers)
90
+ pip install "basa-evals[downstream]"
91
+
92
+ # Full installation including SUT baselines (basa, PySastrawi)
93
+ pip install "basa-evals[basa,sastrawi,tokenizers,downstream,dev]"
94
+ ```
95
+
96
+ ### CLI Commands
97
+
98
+ #### Tier 1: Intrinsic Linguistic Evaluation
99
+ ```bash
100
+ # Run the pilot suite across all registered systems
101
+ basa-eval run --suite pilot_210
102
+
103
+ # Run evaluation on a specific system
104
+ basa-eval run --suite pilot_210 --system basa-default
105
+
106
+ # Run in advisory mode (record results without failing on gate exit code)
107
+ basa-eval run --suite pilot_210 --no-gate
108
+ ```
109
+
110
+ #### Tier 2: Multi-Tokenizer Efficiency Benchmark
111
+ ```bash
112
+ # Run tokenizer benchmark across GPT-4o and LLaMA-3
113
+ basa-eval tokenize --suite tokenizer_pilot --tokenizers gpt-4o,llama-3
114
+
115
+ # Run with offline mock tokenizer (zero dependencies)
116
+ basa-eval tokenize --suite tokenizer_pilot --tokenizers mock-subword
117
+ ```
118
+
119
+ #### Tier 3: Applied Downstream Task Benchmark
120
+ ```bash
121
+ # Run sentiment classification benchmark on IndoNLU SmSA test split
122
+ basa-eval downstream --suite downstream_smsa --classifier mock-sentiment
123
+
124
+ # Export JSON and Markdown benchmark reports
125
+ basa-eval downstream --suite downstream_smsa --classifier mock-sentiment --output-dir reports/
126
+ ```
127
+
128
+ #### Discovery & Inspection Utilities
129
+ ```bash
130
+ # List available suites, systems, tokenizers, and classifiers
131
+ basa-eval list-suites
132
+ basa-eval list-systems
133
+ basa-eval list-tokenizers
134
+ basa-eval list-classifiers
135
+
136
+ # Validate dataset integrity across all slices
137
+ basa-eval validate-dataset
138
+ ```
139
+
140
+ ---
141
+
142
+ ## 5. Explicit Non-Goals & Known Limitations
143
+
144
+ | Limitation | Statement |
145
+ |---|---|
146
+ | **Single-phenomenon scope** | In Tier 1, the six gated slices each target one isolated phenomenon. Multi-phenomenon interactions are measured in the exploratory `mixed_challenge` slice. |
147
+ | **Case invariance** | ONER and entity preservation compare case-folded tokens. Legitimate capitalization changes are monitored via the `EM exact` diagnostic column. |
148
+ | **Punctuation invariance** | Punctuation tokens are ineligible for ONER. Punctuation edits appear in CER/WER only and do not fail the preservation gate. |
149
+ | **Statistical inference** | Benchmark slices in the pilot are curated at 30 rows per phenomenon (150 rows in SmSA). Differences below ~10 percentage points on small slices should be interpreted directionally. |
150
+
151
+ ---
152
+
153
+ ## 6. License & Data Governance
154
+
155
+ - **Software Code:** [MIT License](LICENSE)
156
+ - **Evaluation Slices:** [Creative Commons Attribution 4.0 International (CC-BY-4.0)](https://creativecommons.org/licenses/by/4.0/)
157
+ - All benchmark slices are manually curated or synthetically constructed under explicit provenance. No raw, unconsented social media scrapes are used.
@@ -0,0 +1,16 @@
1
+ name: downstream_smsa
2
+ description: "IndoNLU SmSA customer review sentiment classification benchmark (balanced 150-sample slice)"
3
+ tier: downstream
4
+ dataset_version: v1.0.0
5
+ task: sentiment
6
+ default_classifier: mock-sentiment
7
+
8
+ slices:
9
+ - {path: "downstream/smsa_test_v1.jsonl", slice_name: downstream_sentiment_smsa}
10
+
11
+ systems:
12
+ - {system_id: raw-identity, adapter: raw}
13
+ - {system_id: simple-regex, adapter: simple_regex}
14
+ - {system_id: sastrawi, adapter: sastrawi}
15
+ - {system_id: basa-fast, adapter: basa, params: {lowercase: true, apply_slang: true, apply_typo: false}}
16
+ - {system_id: basa-balanced, adapter: basa, params: {lowercase: true, apply_slang: true, apply_typo: true}}
@@ -0,0 +1,33 @@
1
+ name: pilot_210
2
+ description: "Pilot 210-row intrinsic suite: 6 gated slices + 1 exploratory slice"
3
+ tier: intrinsic
4
+ dataset_version: v1.0.0
5
+
6
+ slices:
7
+ - {path: "challenge/negative_controls_kbbi_v1.jsonl", slice_name: negative_controls_kbbi}
8
+ - {path: "challenge/negative_controls_code_switching_v1.jsonl", slice_name: negative_controls_code_switching}
9
+ - {path: "gold/repetition_reduction_v1.jsonl", slice_name: repetition_reduction}
10
+ - {path: "gold/slang_lexical_v1.jsonl", slice_name: slang_lexical}
11
+ - {path: "gold/abbreviation_phonetic_v1.jsonl", slice_name: abbreviation_phonetic}
12
+ - {path: "gold/typo_nonword_v1.jsonl", slice_name: typo_nonword}
13
+ - {path: "challenge/mixed_challenge_v1.jsonl", slice_name: mixed_challenge, exploratory: true}
14
+
15
+ systems:
16
+ - {system_id: raw-identity, adapter: raw}
17
+ - {system_id: simple-regex, adapter: simple_regex}
18
+ - {system_id: sastrawi, adapter: sastrawi}
19
+ - {system_id: basa-default, adapter: basa, params: {lowercase: true, apply_slang: true, apply_typo: false}}
20
+ - {system_id: basa-full, adapter: basa, params: {lowercase: true, apply_slang: true, apply_typo: true}}
21
+
22
+ gate:
23
+ oner_max_rate: 0.005
24
+ oner_zero_tolerance_below: 500
25
+ per_slice:
26
+ negative_controls_kbbi:
27
+ entity_preservation_min: 1.0
28
+ negative_controls_code_switching:
29
+ entity_preservation_min: 1.0
30
+ repetition_reduction: {uner_max_rate: 0.15}
31
+ slang_lexical: {uner_max_rate: 0.15}
32
+ abbreviation_phonetic: {uner_max_rate: 0.15}
33
+ typo_nonword: {uner_max_rate: 0.15}
@@ -0,0 +1,27 @@
1
+ name: tokenizer_pilot
2
+ description: "Multi-Tokenizer Efficiency Benchmark on the Pilot-210 dataset"
3
+ tier: tokenization
4
+ dataset_version: v1.0.0
5
+
6
+ slices:
7
+ - {path: "challenge/negative_controls_kbbi_v1.jsonl", slice_name: negative_controls_kbbi}
8
+ - {path: "challenge/negative_controls_code_switching_v1.jsonl", slice_name: negative_controls_code_switching}
9
+ - {path: "gold/repetition_reduction_v1.jsonl", slice_name: repetition_reduction}
10
+ - {path: "gold/slang_lexical_v1.jsonl", slice_name: slang_lexical}
11
+ - {path: "gold/abbreviation_phonetic_v1.jsonl", slice_name: abbreviation_phonetic}
12
+ - {path: "gold/typo_nonword_v1.jsonl", slice_name: typo_nonword}
13
+ - {path: "challenge/mixed_challenge_v1.jsonl", slice_name: mixed_challenge, exploratory: true}
14
+
15
+ systems:
16
+ - {system_id: raw-identity, adapter: raw}
17
+ - {system_id: simple-regex, adapter: simple_regex}
18
+ - {system_id: sastrawi, adapter: sastrawi}
19
+ - {system_id: basa-default, adapter: basa, params: {lowercase: true, apply_slang: true, apply_typo: false}}
20
+ - {system_id: basa-full, adapter: basa, params: {lowercase: true, apply_slang: true, apply_typo: true}}
21
+
22
+ tokenizers:
23
+ - gpt-4o
24
+ - llama-3
25
+ - indobert
26
+ - gemma-2
27
+ - mock-subword
@@ -0,0 +1,30 @@
1
+ {"id":"mix-0001","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"iyaaaaa mksd lu apa sih","expected_output":"iya maksud kamu apa sih","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion","lexicon_lookup"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
2
+ {"id":"mix-0002","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"bisaaa tdk kami dpt berkas itu","expected_output":"bisa tidak kami dapat berkas itu","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
3
+ {"id":"mix-0003","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"tidaaaak saya tdk mager hari ini","expected_output":"tidak saya tidak malas gerak hari ini","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion","lexicon_lookup"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
4
+ {"id":"mix-0004","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"apaaa yg km lakukan disana","expected_output":"apa yang kamu lakukan disana","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
5
+ {"id":"mix-0005","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"mana yaaa surat kpd direktur","expected_output":"mana ya surat kepada direktur","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
6
+ {"id":"mix-0006","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"tungguuu kami blm selesai rapat","expected_output":"tunggu kami belum selesai rapat","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
7
+ {"id":"mix-0007","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"baguuus skali hasil kerja tim","expected_output":"bagus sekali hasil kerja tim","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
8
+ {"id":"mix-0008","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"benarrr saya sdh mengirim data","expected_output":"benar saya sudah mengirim data","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
9
+ {"id":"mix-0009","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"sudahhh saya tdk kepo lagi","expected_output":"sudah saya tidak penasaran lagi","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion","lexicon_lookup"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
10
+ {"id":"mix-0010","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"samaaa saja keles penjelasannya","expected_output":"sama saja kali penjelasannya","negative_control":false,"applied_ops":["repetition_collapse","lexicon_lookup"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
11
+ {"id":"mix-0011","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"kerjaaa keras tp tdk dihargai","expected_output":"kerja keras tapi tidak dihargai","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
12
+ {"id":"mix-0012","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"selasaaai sdh tugas akhir kami","expected_output":"selesai sudah tugas akhir kami","negative_control":false,"applied_ops":["repetition_collapse","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
13
+ {"id":"mix-0013","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Penerapan tehknik baru blm selesai.","expected_output":"Penerapan teknik baru belum selesai.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
14
+ {"id":"mix-0014","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"ktika kami sdh sampai di lokasi.","expected_output":"ketika kami sudah sampai di lokasi.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
15
+ {"id":"mix-0015","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Mereka memebuat aplikasi yg sangat mantul.","expected_output":"Mereka membuat aplikasi yang sangat mantap betul.","negative_control":false,"applied_ops":["character_edit","vowel_expansion","lexicon_lookup"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
16
+ {"id":"mix-0016","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Kinerja sistemm bgt cepat.","expected_output":"Kinerja sistem banget cepat.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
17
+ {"id":"mix-0017","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Pusat infomasi tdk buka hari ini.","expected_output":"Pusat informasi tidak buka hari ini.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
18
+ {"id":"mix-0018","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Susun lapran keuangan dgn cermat.","expected_output":"Susun laporan keuangan dengan cermat.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
19
+ {"id":"mix-0019","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Bantuan disalurkan kpd masyrakat luas.","expected_output":"Bantuan disalurkan kepada masyarakat luas.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
20
+ {"id":"mix-0020","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Mutu penddikan di Indonsia msh rendah.","expected_output":"Mutu pendidikan di Indonesia masih rendah.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
21
+ {"id":"mix-0021","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Jaga kebrsihan dlm lingkungan kerja.","expected_output":"Jaga kebersihan dalam lingkungan kerja.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
22
+ {"id":"mix-0022","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Media komunkasi bgt berkembang pesat.","expected_output":"Media komunikasi banget berkembang pesat.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
23
+ {"id":"mix-0023","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Perkembangan tekologi informasi shg mudah diakses.","expected_output":"Perkembangan teknologi informasi sehingga mudah diakses.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
24
+ {"id":"mix-0024","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Laporan keungaan tsb telah diaudit.","expected_output":"Laporan keuangan tersebut telah diaudit.","negative_control":false,"applied_ops":["character_edit","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
25
+ {"id":"mix-0025","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Saya mager krn cuaca dingin.","expected_output":"Saya malas gerak karena cuaca dingin.","negative_control":false,"applied_ops":["lexicon_lookup","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
26
+ {"id":"mix-0026","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Kamu bucin bgt sama dia.","expected_output":"Kamu budak cinta banget sama dia.","negative_control":false,"applied_ops":["lexicon_lookup","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
27
+ {"id":"mix-0027","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Mereka gabut shg bermain game.","expected_output":"Mereka gaji buta sehingga bermain game.","negative_control":false,"applied_ops":["lexicon_lookup","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
28
+ {"id":"mix-0028","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Dia kepo bgt dgn rahasia itu.","expected_output":"Dia penasaran banget dengan rahasia itu.","negative_control":false,"applied_ops":["lexicon_lookup","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
29
+ {"id":"mix-0029","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Pernyataan itu gaje bgt menurut saya.","expected_output":"Pernyataan itu tidak jelas banget menurut saya.","negative_control":false,"applied_ops":["lexicon_lookup","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
30
+ {"id":"mix-0030","dataset_version":"v1.0.0","slice":"mixed_challenge","domain":"social_chat","raw_input":"Hasil kerja mantul bgt utk dievaluasi.","expected_output":"Hasil kerja mantap betul banget untuk dievaluasi.","negative_control":false,"applied_ops":["lexicon_lookup","vowel_expansion"],"metadata":{"protected_entities":[],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
@@ -0,0 +1,30 @@
1
+ {"id":"neg-cs-0001","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Tim developer sedang melakukan deploy ke lingkungan staging.","expected_output":"Tim developer sedang melakukan deploy ke lingkungan staging.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["deploy","staging"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
2
+ {"id":"neg-cs-0002","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Arsitek sistem memantau performa di lingkungan production.","expected_output":"Arsitek sistem memantau performa di lingkungan production.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["production"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
3
+ {"id":"neg-cs-0003","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Seluruh tugas telah dimasukkan ke dalam sprint backlog.","expected_output":"Seluruh tugas telah dimasukkan ke dalam sprint backlog.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["sprint","backlog"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
4
+ {"id":"neg-cs-0004","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Kami menjadwalkan meeting sinkronisasi setiap pagi.","expected_output":"Kami menjadwalkan meeting sinkronisasi setiap pagi.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["meeting"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
5
+ {"id":"neg-cs-0005","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Proyek harus selesai sebelum deadline akhir bulan.","expected_output":"Proyek harus selesai sebelum deadline akhir bulan.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["deadline"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
6
+ {"id":"neg-cs-0006","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Perubahan ini berada di luar scope proyek awal.","expected_output":"Perubahan ini berada di luar scope proyek awal.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["scope"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
7
+ {"id":"neg-cs-0007","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Operator melakukan rollback karena terdapat kesalahan konfigurasi.","expected_output":"Operator melakukan rollback karena terdapat kesalahan konfigurasi.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["rollback"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
8
+ {"id":"neg-cs-0008","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Pastikan setiap commit memiliki pesan yang jelas.","expected_output":"Pastikan setiap commit memiliki pesan yang jelas.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["commit"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
9
+ {"id":"neg-cs-0009","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Fitur baru dikembangkan pada branch terpisah.","expected_output":"Fitur baru dikembangkan pada branch terpisah.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["branch"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
10
+ {"id":"neg-cs-0010","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Tech lead melakukan merge setelah tinjauan kode selesai.","expected_output":"Tech lead melakukan merge setelah tinjauan kode selesai.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["merge"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
11
+ {"id":"neg-cs-0011","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Layanan microservice dijalankan menggunakan container Docker.","expected_output":"Layanan microservice dijalankan menggunakan container Docker.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["Docker"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
12
+ {"id":"neg-cs-0012","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Pipa CI/CD secara otomatis menguji seluruh kode baru.","expected_output":"Pipa CI/CD secara otomatis menguji seluruh kode baru.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["CI/CD"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
13
+ {"id":"neg-cs-0013","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Cache Redis digunakan untuk mempercepat respons kueri.","expected_output":"Cache Redis digunakan untuk mempercepat respons kueri.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["Redis"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
14
+ {"id":"neg-cs-0014","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Koneksi database PostgreSQL dilindungi dengan otentikasi ketat.","expected_output":"Koneksi database PostgreSQL dilindungi dengan otentikasi ketat.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["PostgreSQL"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
15
+ {"id":"neg-cs-0015","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Pesan acara dikirimkan melalui broker Apache Kafka.","expected_output":"Pesan acara dikirimkan melalui broker Apache Kafka.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["Kafka"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
16
+ {"id":"neg-cs-0016","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Infrastruktur komputasi awan dikelola dengan Terraform.","expected_output":"Infrastruktur komputasi awan dikelola dengan Terraform.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["Terraform"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
17
+ {"id":"neg-cs-0017","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Dokumentasi API tersedia di Swagger UI.","expected_output":"Dokumentasi API tersedia di Swagger UI.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["Swagger UI"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
18
+ {"id":"neg-cs-0018","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Pengujian integrasi dijalankan sebelum rilis ke production.","expected_output":"Pengujian integrasi dijalankan sebelum rilis ke production.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["production"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
19
+ {"id":"neg-cs-0019","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Tim security melakukan audit berkala terhadap endpoint publik.","expected_output":"Tim security melakukan audit berkala terhadap endpoint publik.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["security","endpoint"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
20
+ {"id":"neg-cs-0020","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Sistem logging terpusat memantau setiap exception aplikasi.","expected_output":"Sistem logging terpusat memantau setiap exception aplikasi.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["logging","exception"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
21
+ {"id":"neg-cs-0021","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Pengaturan load balancer mendistribusikan lalu lintas secara merata.","expected_output":"Pengaturan load balancer mendistribusikan lalu lintas secara merata.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["load balancer"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
22
+ {"id":"neg-cs-0022","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Konfigurasi webhook mengirimkan notifikasi ke kanal Slack.","expected_output":"Konfigurasi webhook mengirimkan notifikasi ke kanal Slack.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["webhook","Slack"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
23
+ {"id":"neg-cs-0023","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Proses build container selesai dalam waktu tiga menit.","expected_output":"Proses build container selesai dalam waktu tiga menit.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["build"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
24
+ {"id":"neg-cs-0024","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Skrip deployment dieksekusi secara otomatis oleh runner.","expected_output":"Skrip deployment dieksekusi secara otomatis oleh runner.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["deployment","runner"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
25
+ {"id":"neg-cs-0025","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Metrik throughput dipantau menggunakan dashboard Grafana.","expected_output":"Metrik throughput dipantau menggunakan dashboard Grafana.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["throughput","Grafana"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
26
+ {"id":"neg-cs-0026","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Proses onboarding anggota tim baru berjalan lancar.","expected_output":"Proses onboarding anggota tim baru berjalan lancar.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["onboarding"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
27
+ {"id":"neg-cs-0027","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Dokumentasi teknis disimpan di repositori Git internal.","expected_output":"Dokumentasi teknis disimpan di repositori Git internal.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["Git"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
28
+ {"id":"neg-cs-0028","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Server proxy menyaring permintaan masuk yang mencurigakan.","expected_output":"Server proxy menyaring permintaan masuk yang mencurigakan.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["proxy"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
29
+ {"id":"neg-cs-0029","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Pengembang mengoptimalkan indeks database untuk mengurangi latency.","expected_output":"Pengembang mengoptimalkan indeks database untuk mengurangi latency.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["latency"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}
30
+ {"id":"neg-cs-0030","dataset_version":"v1.0.0","slice":"negative_controls_code_switching","domain":"tech_workplace","raw_input":"Sesi brainstorming menghasilkan beberapa ide fitur inovatif.","expected_output":"Sesi brainstorming menghasilkan beberapa ide fitur inovatif.","negative_control":true,"applied_ops":[],"metadata":{"protected_entities":["brainstorming"],"source":"curated_pilot_v2","difficulty":"medium","license":"CC-BY-4.0","extra":{}}}