satuasummarization 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. satuasummarization-0.1.0/LICENSE +28 -0
  2. satuasummarization-0.1.0/PKG-INFO +163 -0
  3. satuasummarization-0.1.0/README.md +135 -0
  4. satuasummarization-0.1.0/pyproject.toml +39 -0
  5. satuasummarization-0.1.0/satuasummarization/__init__.py +2 -0
  6. satuasummarization-0.1.0/satuasummarization/abstractive/__init__.py +1 -0
  7. satuasummarization-0.1.0/satuasummarization/extractive/__init__.py +14 -0
  8. satuasummarization-0.1.0/satuasummarization/extractive/data.py +52 -0
  9. satuasummarization-0.1.0/satuasummarization/extractive/features.py +177 -0
  10. satuasummarization-0.1.0/satuasummarization/extractive/metrics.py +101 -0
  11. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/AHO.json +218 -0
  12. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/AO.json +218 -0
  13. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/AOA.json +218 -0
  14. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/APO.json +218 -0
  15. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/BEO.json +144 -0
  16. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/CFO.json +169 -0
  17. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/CFOA.json +218 -0
  18. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/DIO.json +218 -0
  19. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/DRA.json +218 -0
  20. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/ECO.json +218 -0
  21. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/EEGWO.json +218 -0
  22. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/EGO.json +169 -0
  23. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/EPO.json +218 -0
  24. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/FATA.json +218 -0
  25. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/FLA.json +218 -0
  26. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/GA.json +226 -0
  27. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/GCRA.json +188 -0
  28. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/GTO.json +169 -0
  29. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/GWO.json +218 -0
  30. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/GWOWOA.json +218 -0
  31. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/HGS.json +218 -0
  32. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/HGWODE.json +169 -0
  33. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/HHO.json +180 -0
  34. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/HO.json +152 -0
  35. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/HOA.json +218 -0
  36. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/IAKMA.json +218 -0
  37. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/IGWO.json +218 -0
  38. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/INFO.json +218 -0
  39. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/KMA.json +218 -0
  40. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/MBKA.json +169 -0
  41. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/MGO.json +218 -0
  42. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/MGWO.json +218 -0
  43. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/MO.json +218 -0
  44. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/PFA.json +127 -0
  45. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/PLO.json +218 -0
  46. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/PO.json +218 -0
  47. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/PSO.json +218 -0
  48. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/PSOGWO.json +218 -0
  49. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/PUMA.json +215 -0
  50. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/RIME.json +218 -0
  51. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/RUN.json +178 -0
  52. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/SAR.json +168 -0
  53. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/SMA.json +218 -0
  54. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/SSA.json +218 -0
  55. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/WOA.json +218 -0
  56. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/WaOA.json +152 -0
  57. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/index.json +679 -0
  58. satuasummarization-0.1.0/satuasummarization/extractive/pretrained_optimal_models/mEGO.json +169 -0
  59. satuasummarization-0.1.0/satuasummarization/extractive/summarizer.py +163 -0
  60. satuasummarization-0.1.0/satuasummarization.egg-info/PKG-INFO +163 -0
  61. satuasummarization-0.1.0/satuasummarization.egg-info/SOURCES.txt +64 -0
  62. satuasummarization-0.1.0/satuasummarization.egg-info/dependency_links.txt +1 -0
  63. satuasummarization-0.1.0/satuasummarization.egg-info/requires.txt +13 -0
  64. satuasummarization-0.1.0/satuasummarization.egg-info/top_level.txt +1 -0
  65. satuasummarization-0.1.0/setup.cfg +4 -0
  66. satuasummarization-0.1.0/tests/test_extractive.py +125 -0
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2026, I Made Satria Bimantara
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,163 @@
1
+ Metadata-Version: 2.4
2
+ Name: satuasummarization
3
+ Version: 0.1.0
4
+ Summary: Balinese extractive summarisation with metaheuristic-optimised sentence feature weights
5
+ Author: I Made Satria Bimantara
6
+ License-Expression: BSD-3-Clause
7
+ Project-URL: Homepage, https://github.com/satriabimantara/balinese-aisummarizer
8
+ Keywords: balinese,summarization,extractive summarization,metaheuristic,nlp,satua
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: Programming Language :: Python :: 3.11
11
+ Classifier: Natural Language :: Indonesian
12
+ Classifier: Topic :: Text Processing :: Linguistic
13
+ Classifier: Development Status :: 3 - Alpha
14
+ Requires-Python: >=3.10
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Requires-Dist: numpy>=1.24
18
+ Provides-Extra: nlp
19
+ Requires-Dist: balinese-nlp==2.4.2; extra == "nlp"
20
+ Provides-Extra: test
21
+ Requires-Dist: pytest>=8; extra == "test"
22
+ Provides-Extra: train
23
+ Requires-Dist: metaheuristicpy>=3.2.0; extra == "train"
24
+ Requires-Dist: pandas; extra == "train"
25
+ Requires-Dist: openpyxl; extra == "train"
26
+ Requires-Dist: joblib; extra == "train"
27
+ Dynamic: license-file
28
+
29
+ # balinese-aisummarizer · `satuasummarization`
30
+ Artificial Intelligence Models for Balinese Scripts Summarization.
31
+
32
+ Paket Python peringkasan teks berbahasa Bali untuk Modul 3 SatuaGen-AI (proyek Dana Indonesia Raya 2026).
33
+
34
+ | Sub-paket | Isi | Status |
35
+ |---|---|---|
36
+ | `satuasummarization.extractive` | Peringkasan ekstraktif: bobot fitur kalimat dioptimasi metaheuristik (47 algoritme dari `metaheuristicpy`) | Tersedia, beserta 47 model pretrained |
37
+ | `satuasummarization.abstractive` | Peringkasan abstraktif berbasis LLM | Dicadangkan (kosong) |
38
+
39
+ Penulis: I Made Satria Bimantara.
40
+
41
+ ## Cara kerja (ekstraktif)
42
+
43
+ 1. Setiap kalimat dokumen diubah menjadi 13 fitur yang dinormalisasi min-max di dalam dokumen
44
+ (`SentenceFeatureExtractor`): panjang, posisi, rasio angka, Jaccard judul, rerata TF-ISF, kosinus ke
45
+ sentroid dan ke judul, kemiripan antarkalimat, rasio dialog, kepadatan nomina/verba/nama diri (POS HMM
46
+ `balinese-nlp`), dan kepadatan tokoh (SatuaNER).
47
+ 2. Skor kalimat `score(s) = w · x(s)`, dengan `w ∈ [0,1]^13` hasil optimasi metaheuristik.
48
+ 3. `k = ceil(rasio · n)` kalimat berskor tertinggi diambil dan disusun menurut urutan aslinya.
49
+ 4. Pelatihan memaksimalkan rerata (ROUGE-1 F1 + ROUGE-2 F1)/2 pada dokumen latih, dengan `k` sama dengan jumlah
50
+ kalimat referensi. Semua algoritme memakai anggaran evaluasi yang sama (30 agen × 100 iterasi, 3 seed).
51
+ 5. Model `best` dipilih dengan 5-fold cross-validation di dalam data latih, bukan dari data uji.
52
+
53
+ ## Instalasi
54
+
55
+ ```bash
56
+ python -m pip install "satuasummarization[nlp]" # inferensi (POS + SatuaNER dari balinese-nlp 2.4.2)
57
+ python -m pip install "satuasummarization[train]" # hanya untuk melatih ulang (metaheuristicpy >= 3.2.0)
58
+ ```
59
+
60
+ ## Pemakaian
61
+
62
+ ```python
63
+ from satuasummarization.extractive import MetaheuristicSummarizer, available_models
64
+
65
+ model = MetaheuristicSummarizer.from_pretrained("best") # model terbaik (lihat index.json)
66
+ out = model.summarize(teks, title="I Kancil Nulungin I Kakua", ratio=0.3)
67
+ print(out["summary"]) # kalimat terpilih
68
+ out["sentences"] # [{'index', 'text', 'score', 'selected'}, ...]
69
+
70
+ MetaheuristicSummarizer.from_pretrained("GWO") # model metaheuristik tertentu
71
+ available_models()["models"]["GWO"] # metrik uji model tersebut
72
+ ```
73
+
74
+ Melatih dengan data sendiri:
75
+
76
+ ```python
77
+ from satuasummarization.extractive import SentenceFeatureExtractor, load_corpus, train_test_split_docs
78
+ docs = load_corpus("datasets/BalineseSummarizationDatasets/extractive") # .xlsx: sentences, extractive_summary
79
+ fx = SentenceFeatureExtractor()
80
+ for d in docs:
81
+ d.features = fx.transform(d.sentences, d.title)
82
+ train, test = train_test_split_docs(docs, test_size=0.2, seed=42)
83
+ m = MetaheuristicSummarizer(ratio=0.4).fit(train, algorithm="PLO", n_agents=30, max_iter=100, seed=0)
84
+ m.evaluate(test) # ROUGE-1/2/L P/R/F1, akurasi, P/R/F1 makro kalimat
85
+ m.save("plo.json")
86
+ ```
87
+
88
+ Model pretrained berupa JSON (bobot + metadata), bukan pickle. `from_pretrained` memeriksa sha256 setiap berkas
89
+ terhadap `pretrained_optimal_models/index.json` sebelum dipakai.
90
+
91
+ ## Reproduksi
92
+
93
+ ```bash
94
+ python scripts/extract_features.py # 463 dokumen, 20.029 kalimat -> results/corpus_features.pkl (±30 menit, SatuaNER)
95
+ python scripts/train_pretrained.py # 47 algoritme x 3 seed -> pretrained_optimal_models/ + results/pretrained_results.csv
96
+ python -m pytest # uji paket
97
+ ```
98
+
99
+ ## Hasil
100
+
101
+ Pemilihan model: 5-fold cross-validation di dalam 80% data latih (370 dokumen); skor CV = rerata ROUGE-1/2/L F1
102
+ pada lipatan validasi. Model `best` adalah model dengan skor CV tertinggi. Data uji (93 dokumen, 20% per sumber,
103
+ seed 42) hanya dipakai untuk pelaporan. Bobot yang dirilis dilatih pada seluruh data latih (3 seed, fitness latih
104
+ terbaik). Panjang ringkasan saat evaluasi = jumlah kalimat referensi. Sumber: `results/pretrained_results.csv`.
105
+
106
+ | # | Model | Skor CV (5-fold) | ROUGE-1 F1 uji | ROUGE-2 F1 uji | ROUGE-L F1 uji | F1 makro kalimat | Waktu latih / seed (s) |
107
+ |---:|---|---:|---:|---:|---:|---:|---:|
108
+ | 1 | APO **(best)** | 0.6719 | 0.7363 | 0.6726 | 0.6776 | 0.6246 | 23 |
109
+ | 2 | MGWO | 0.6711 | 0.7386 | 0.6769 | 0.6808 | 0.6264 | 28 |
110
+ | 3 | GWOWOA | 0.6707 | 0.7368 | 0.6759 | 0.6784 | 0.6307 | 24 |
111
+ | 4 | GWO | 0.6706 | 0.7334 | 0.6713 | 0.6752 | 0.6272 | 25 |
112
+ | 5 | SMA | 0.6704 | 0.7343 | 0.6726 | 0.6746 | 0.6201 | 20 |
113
+ | 6 | mEGO | 0.6702 | 0.7385 | 0.6788 | 0.6829 | 0.6305 | 116 |
114
+ | 7 | MGO | 0.6697 | 0.7311 | 0.6697 | 0.6744 | 0.6178 | 24 |
115
+ | 8 | EPO | 0.6695 | 0.7296 | 0.6681 | 0.6692 | 0.6165 | 27 |
116
+ | 9 | IAKMA | 0.6693 | 0.7313 | 0.6696 | 0.6729 | 0.6174 | 24 |
117
+ | 10 | DRA | 0.6692 | 0.7331 | 0.6700 | 0.6741 | 0.6204 | 207 |
118
+ | 11 | MO | 0.6689 | 0.7270 | 0.6645 | 0.6679 | 0.6110 | 23 |
119
+ | 12 | SSA | 0.6686 | 0.7243 | 0.6608 | 0.6656 | 0.6127 | 21 |
120
+ | 13 | GTO | 0.6684 | 0.7283 | 0.6653 | 0.6673 | 0.6116 | 22 |
121
+ | 14 | CFOA | 0.6684 | 0.7336 | 0.6740 | 0.6768 | 0.6249 | 24 |
122
+ | 15 | IGWO | 0.6679 | 0.7371 | 0.6775 | 0.6808 | 0.6271 | 21 |
123
+ | 16 | GA | 0.6679 | 0.7355 | 0.6723 | 0.6774 | 0.6234 | 23 |
124
+ | 17 | PSOGWO | 0.6679 | 0.7369 | 0.6758 | 0.6793 | 0.6290 | 23 |
125
+ | 18 | FLA | 0.6672 | 0.7367 | 0.6749 | 0.6792 | 0.6272 | 228 |
126
+ | 19 | EGO | 0.6667 | 0.7342 | 0.6726 | 0.6742 | 0.6242 | 134 |
127
+ | 20 | WaOA | 0.6663 | 0.7306 | 0.6693 | 0.6725 | 0.6184 | 24 |
128
+ | 21 | HGS | 0.6659 | 0.7331 | 0.6699 | 0.6744 | 0.6197 | 24 |
129
+ | 22 | FATA | 0.6657 | 0.7345 | 0.6739 | 0.6774 | 0.6258 | 24 |
130
+ | 23 | HGWODE | 0.6657 | 0.7375 | 0.6768 | 0.6797 | 0.6259 | 26 |
131
+ | 24 | RIME | 0.6656 | 0.7307 | 0.6679 | 0.6728 | 0.6200 | 24 |
132
+ | 25 | AOA | 0.6656 | 0.7368 | 0.6734 | 0.6786 | 0.6245 | 21 |
133
+ | 26 | PO | 0.6655 | 0.7416 | 0.6808 | 0.6819 | 0.6333 | 25 |
134
+ | 27 | ECO | 0.6652 | 0.7295 | 0.6665 | 0.6723 | 0.6171 | 255 |
135
+ | 28 | AO | 0.6646 | 0.7334 | 0.6730 | 0.6761 | 0.6246 | 259 |
136
+ | 29 | PFA | 0.6646 | 0.7341 | 0.6716 | 0.6754 | 0.6239 | 226 |
137
+ | 30 | SAR | 0.6643 | 0.7370 | 0.6746 | 0.6793 | 0.6282 | 257 |
138
+ | 31 | MBKA | 0.6641 | 0.7313 | 0.6661 | 0.6729 | 0.6182 | 216 |
139
+ | 32 | CFO | 0.6640 | 0.7356 | 0.6719 | 0.6743 | 0.6237 | 24 |
140
+ | 33 | RUN | 0.6638 | 0.7309 | 0.6695 | 0.6735 | 0.6184 | 260 |
141
+ | 34 | BEO | 0.6637 | 0.7333 | 0.6708 | 0.6745 | 0.6218 | 22 |
142
+ | 35 | DIO | 0.6636 | 0.7353 | 0.6755 | 0.6789 | 0.6286 | 23 |
143
+ | 36 | HO | 0.6633 | 0.7290 | 0.6683 | 0.6702 | 0.6174 | 230 |
144
+ | 37 | PSO | 0.6632 | 0.7388 | 0.6790 | 0.6824 | 0.6345 | 24 |
145
+ | 38 | KMA | 0.6630 | 0.7295 | 0.6649 | 0.6715 | 0.6159 | 25 |
146
+ | 39 | WOA | 0.6629 | 0.7256 | 0.6646 | 0.6676 | 0.6129 | 20 |
147
+ | 40 | HHO | 0.6627 | 0.7361 | 0.6781 | 0.6802 | 0.6304 | 115 |
148
+ | 41 | GCRA | 0.6627 | 0.7299 | 0.6691 | 0.6707 | 0.6185 | 256 |
149
+ | 42 | PLO | 0.6619 | 0.7307 | 0.6700 | 0.6731 | 0.6219 | 24 |
150
+ | 43 | PUMA | 0.6618 | 0.7363 | 0.6763 | 0.6810 | 0.6293 | 238 |
151
+ | 44 | HOA | 0.6608 | 0.7262 | 0.6627 | 0.6671 | 0.6105 | 227 |
152
+ | 45 | AHO | 0.6600 | 0.7289 | 0.6660 | 0.6687 | 0.6145 | 22 |
153
+ | 46 | INFO | 0.6594 | 0.7320 | 0.6678 | 0.6734 | 0.6185 | 255 |
154
+ | 47 | EEGWO | 0.6584 | 0.7331 | 0.6732 | 0.6748 | 0.6250 | 23 |
155
+ | – | Lead-k (baseline) | – | 0.6398 | 0.5501 | 0.5428 | 0.5543 | – |
156
+ | – | EqualWeights (baseline) | – | 0.7144 | 0.6464 | 0.6515 | 0.6006 | – |
157
+ | – | RandomWeights (baseline) | – | 0.7174 | 0.6509 | 0.6551 | 0.6020 | – |
158
+
159
+ Catatan: semua model metaheuristik lebih baik daripada ketiga baseline, tetapi selisih antaralgoritme kecil
160
+ (ROUGE-1 F1 uji 0.7243–0.7416; korelasi skor CV dengan ROUGE-1 uji hanya 0.21),
161
+ sehingga peringkat antaralgoritme tidak boleh ditafsirkan sebagai perbedaan yang bermakna.
162
+
163
+ Pengujian paket: `python -m pytest` → 10 tes lulus.
@@ -0,0 +1,135 @@
1
+ # balinese-aisummarizer · `satuasummarization`
2
+ Artificial Intelligence Models for Balinese Scripts Summarization.
3
+
4
+ Paket Python peringkasan teks berbahasa Bali untuk Modul 3 SatuaGen-AI (proyek Dana Indonesia Raya 2026).
5
+
6
+ | Sub-paket | Isi | Status |
7
+ |---|---|---|
8
+ | `satuasummarization.extractive` | Peringkasan ekstraktif: bobot fitur kalimat dioptimasi metaheuristik (47 algoritme dari `metaheuristicpy`) | Tersedia, beserta 47 model pretrained |
9
+ | `satuasummarization.abstractive` | Peringkasan abstraktif berbasis LLM | Dicadangkan (kosong) |
10
+
11
+ Penulis: I Made Satria Bimantara.
12
+
13
+ ## Cara kerja (ekstraktif)
14
+
15
+ 1. Setiap kalimat dokumen diubah menjadi 13 fitur yang dinormalisasi min-max di dalam dokumen
16
+ (`SentenceFeatureExtractor`): panjang, posisi, rasio angka, Jaccard judul, rerata TF-ISF, kosinus ke
17
+ sentroid dan ke judul, kemiripan antarkalimat, rasio dialog, kepadatan nomina/verba/nama diri (POS HMM
18
+ `balinese-nlp`), dan kepadatan tokoh (SatuaNER).
19
+ 2. Skor kalimat `score(s) = w · x(s)`, dengan `w ∈ [0,1]^13` hasil optimasi metaheuristik.
20
+ 3. `k = ceil(rasio · n)` kalimat berskor tertinggi diambil dan disusun menurut urutan aslinya.
21
+ 4. Pelatihan memaksimalkan rerata (ROUGE-1 F1 + ROUGE-2 F1)/2 pada dokumen latih, dengan `k` sama dengan jumlah
22
+ kalimat referensi. Semua algoritme memakai anggaran evaluasi yang sama (30 agen × 100 iterasi, 3 seed).
23
+ 5. Model `best` dipilih dengan 5-fold cross-validation di dalam data latih, bukan dari data uji.
24
+
25
+ ## Instalasi
26
+
27
+ ```bash
28
+ python -m pip install "satuasummarization[nlp]" # inferensi (POS + SatuaNER dari balinese-nlp 2.4.2)
29
+ python -m pip install "satuasummarization[train]" # hanya untuk melatih ulang (metaheuristicpy >= 3.2.0)
30
+ ```
31
+
32
+ ## Pemakaian
33
+
34
+ ```python
35
+ from satuasummarization.extractive import MetaheuristicSummarizer, available_models
36
+
37
+ model = MetaheuristicSummarizer.from_pretrained("best") # model terbaik (lihat index.json)
38
+ out = model.summarize(teks, title="I Kancil Nulungin I Kakua", ratio=0.3)
39
+ print(out["summary"]) # kalimat terpilih
40
+ out["sentences"] # [{'index', 'text', 'score', 'selected'}, ...]
41
+
42
+ MetaheuristicSummarizer.from_pretrained("GWO") # model metaheuristik tertentu
43
+ available_models()["models"]["GWO"] # metrik uji model tersebut
44
+ ```
45
+
46
+ Melatih dengan data sendiri:
47
+
48
+ ```python
49
+ from satuasummarization.extractive import SentenceFeatureExtractor, load_corpus, train_test_split_docs
50
+ docs = load_corpus("datasets/BalineseSummarizationDatasets/extractive") # .xlsx: sentences, extractive_summary
51
+ fx = SentenceFeatureExtractor()
52
+ for d in docs:
53
+ d.features = fx.transform(d.sentences, d.title)
54
+ train, test = train_test_split_docs(docs, test_size=0.2, seed=42)
55
+ m = MetaheuristicSummarizer(ratio=0.4).fit(train, algorithm="PLO", n_agents=30, max_iter=100, seed=0)
56
+ m.evaluate(test) # ROUGE-1/2/L P/R/F1, akurasi, P/R/F1 makro kalimat
57
+ m.save("plo.json")
58
+ ```
59
+
60
+ Model pretrained berupa JSON (bobot + metadata), bukan pickle. `from_pretrained` memeriksa sha256 setiap berkas
61
+ terhadap `pretrained_optimal_models/index.json` sebelum dipakai.
62
+
63
+ ## Reproduksi
64
+
65
+ ```bash
66
+ python scripts/extract_features.py # 463 dokumen, 20.029 kalimat -> results/corpus_features.pkl (±30 menit, SatuaNER)
67
+ python scripts/train_pretrained.py # 47 algoritme x 3 seed -> pretrained_optimal_models/ + results/pretrained_results.csv
68
+ python -m pytest # uji paket
69
+ ```
70
+
71
+ ## Hasil
72
+
73
+ Pemilihan model: 5-fold cross-validation di dalam 80% data latih (370 dokumen); skor CV = rerata ROUGE-1/2/L F1
74
+ pada lipatan validasi. Model `best` adalah model dengan skor CV tertinggi. Data uji (93 dokumen, 20% per sumber,
75
+ seed 42) hanya dipakai untuk pelaporan. Bobot yang dirilis dilatih pada seluruh data latih (3 seed, fitness latih
76
+ terbaik). Panjang ringkasan saat evaluasi = jumlah kalimat referensi. Sumber: `results/pretrained_results.csv`.
77
+
78
+ | # | Model | Skor CV (5-fold) | ROUGE-1 F1 uji | ROUGE-2 F1 uji | ROUGE-L F1 uji | F1 makro kalimat | Waktu latih / seed (s) |
79
+ |---:|---|---:|---:|---:|---:|---:|---:|
80
+ | 1 | APO **(best)** | 0.6719 | 0.7363 | 0.6726 | 0.6776 | 0.6246 | 23 |
81
+ | 2 | MGWO | 0.6711 | 0.7386 | 0.6769 | 0.6808 | 0.6264 | 28 |
82
+ | 3 | GWOWOA | 0.6707 | 0.7368 | 0.6759 | 0.6784 | 0.6307 | 24 |
83
+ | 4 | GWO | 0.6706 | 0.7334 | 0.6713 | 0.6752 | 0.6272 | 25 |
84
+ | 5 | SMA | 0.6704 | 0.7343 | 0.6726 | 0.6746 | 0.6201 | 20 |
85
+ | 6 | mEGO | 0.6702 | 0.7385 | 0.6788 | 0.6829 | 0.6305 | 116 |
86
+ | 7 | MGO | 0.6697 | 0.7311 | 0.6697 | 0.6744 | 0.6178 | 24 |
87
+ | 8 | EPO | 0.6695 | 0.7296 | 0.6681 | 0.6692 | 0.6165 | 27 |
88
+ | 9 | IAKMA | 0.6693 | 0.7313 | 0.6696 | 0.6729 | 0.6174 | 24 |
89
+ | 10 | DRA | 0.6692 | 0.7331 | 0.6700 | 0.6741 | 0.6204 | 207 |
90
+ | 11 | MO | 0.6689 | 0.7270 | 0.6645 | 0.6679 | 0.6110 | 23 |
91
+ | 12 | SSA | 0.6686 | 0.7243 | 0.6608 | 0.6656 | 0.6127 | 21 |
92
+ | 13 | GTO | 0.6684 | 0.7283 | 0.6653 | 0.6673 | 0.6116 | 22 |
93
+ | 14 | CFOA | 0.6684 | 0.7336 | 0.6740 | 0.6768 | 0.6249 | 24 |
94
+ | 15 | IGWO | 0.6679 | 0.7371 | 0.6775 | 0.6808 | 0.6271 | 21 |
95
+ | 16 | GA | 0.6679 | 0.7355 | 0.6723 | 0.6774 | 0.6234 | 23 |
96
+ | 17 | PSOGWO | 0.6679 | 0.7369 | 0.6758 | 0.6793 | 0.6290 | 23 |
97
+ | 18 | FLA | 0.6672 | 0.7367 | 0.6749 | 0.6792 | 0.6272 | 228 |
98
+ | 19 | EGO | 0.6667 | 0.7342 | 0.6726 | 0.6742 | 0.6242 | 134 |
99
+ | 20 | WaOA | 0.6663 | 0.7306 | 0.6693 | 0.6725 | 0.6184 | 24 |
100
+ | 21 | HGS | 0.6659 | 0.7331 | 0.6699 | 0.6744 | 0.6197 | 24 |
101
+ | 22 | FATA | 0.6657 | 0.7345 | 0.6739 | 0.6774 | 0.6258 | 24 |
102
+ | 23 | HGWODE | 0.6657 | 0.7375 | 0.6768 | 0.6797 | 0.6259 | 26 |
103
+ | 24 | RIME | 0.6656 | 0.7307 | 0.6679 | 0.6728 | 0.6200 | 24 |
104
+ | 25 | AOA | 0.6656 | 0.7368 | 0.6734 | 0.6786 | 0.6245 | 21 |
105
+ | 26 | PO | 0.6655 | 0.7416 | 0.6808 | 0.6819 | 0.6333 | 25 |
106
+ | 27 | ECO | 0.6652 | 0.7295 | 0.6665 | 0.6723 | 0.6171 | 255 |
107
+ | 28 | AO | 0.6646 | 0.7334 | 0.6730 | 0.6761 | 0.6246 | 259 |
108
+ | 29 | PFA | 0.6646 | 0.7341 | 0.6716 | 0.6754 | 0.6239 | 226 |
109
+ | 30 | SAR | 0.6643 | 0.7370 | 0.6746 | 0.6793 | 0.6282 | 257 |
110
+ | 31 | MBKA | 0.6641 | 0.7313 | 0.6661 | 0.6729 | 0.6182 | 216 |
111
+ | 32 | CFO | 0.6640 | 0.7356 | 0.6719 | 0.6743 | 0.6237 | 24 |
112
+ | 33 | RUN | 0.6638 | 0.7309 | 0.6695 | 0.6735 | 0.6184 | 260 |
113
+ | 34 | BEO | 0.6637 | 0.7333 | 0.6708 | 0.6745 | 0.6218 | 22 |
114
+ | 35 | DIO | 0.6636 | 0.7353 | 0.6755 | 0.6789 | 0.6286 | 23 |
115
+ | 36 | HO | 0.6633 | 0.7290 | 0.6683 | 0.6702 | 0.6174 | 230 |
116
+ | 37 | PSO | 0.6632 | 0.7388 | 0.6790 | 0.6824 | 0.6345 | 24 |
117
+ | 38 | KMA | 0.6630 | 0.7295 | 0.6649 | 0.6715 | 0.6159 | 25 |
118
+ | 39 | WOA | 0.6629 | 0.7256 | 0.6646 | 0.6676 | 0.6129 | 20 |
119
+ | 40 | HHO | 0.6627 | 0.7361 | 0.6781 | 0.6802 | 0.6304 | 115 |
120
+ | 41 | GCRA | 0.6627 | 0.7299 | 0.6691 | 0.6707 | 0.6185 | 256 |
121
+ | 42 | PLO | 0.6619 | 0.7307 | 0.6700 | 0.6731 | 0.6219 | 24 |
122
+ | 43 | PUMA | 0.6618 | 0.7363 | 0.6763 | 0.6810 | 0.6293 | 238 |
123
+ | 44 | HOA | 0.6608 | 0.7262 | 0.6627 | 0.6671 | 0.6105 | 227 |
124
+ | 45 | AHO | 0.6600 | 0.7289 | 0.6660 | 0.6687 | 0.6145 | 22 |
125
+ | 46 | INFO | 0.6594 | 0.7320 | 0.6678 | 0.6734 | 0.6185 | 255 |
126
+ | 47 | EEGWO | 0.6584 | 0.7331 | 0.6732 | 0.6748 | 0.6250 | 23 |
127
+ | – | Lead-k (baseline) | – | 0.6398 | 0.5501 | 0.5428 | 0.5543 | – |
128
+ | – | EqualWeights (baseline) | – | 0.7144 | 0.6464 | 0.6515 | 0.6006 | – |
129
+ | – | RandomWeights (baseline) | – | 0.7174 | 0.6509 | 0.6551 | 0.6020 | – |
130
+
131
+ Catatan: semua model metaheuristik lebih baik daripada ketiga baseline, tetapi selisih antaralgoritme kecil
132
+ (ROUGE-1 F1 uji 0.7243–0.7416; korelasi skor CV dengan ROUGE-1 uji hanya 0.21),
133
+ sehingga peringkat antaralgoritme tidak boleh ditafsirkan sebagai perbedaan yang bermakna.
134
+
135
+ Pengujian paket: `python -m pytest` → 10 tes lulus.
@@ -0,0 +1,39 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "satuasummarization"
7
+ version = "0.1.0"
8
+ description = "Balinese extractive summarisation with metaheuristic-optimised sentence feature weights"
9
+ authors = [{ name = "I Made Satria Bimantara" }]
10
+ readme = "README.md"
11
+ license = "BSD-3-Clause"
12
+ license-files = ["LICENSE"]
13
+ requires-python = ">=3.10"
14
+ keywords = ["balinese", "summarization", "extractive summarization", "metaheuristic", "nlp", "satua"]
15
+ classifiers = [
16
+ "Programming Language :: Python :: 3",
17
+ "Programming Language :: Python :: 3.11",
18
+ "Natural Language :: Indonesian",
19
+ "Topic :: Text Processing :: Linguistic",
20
+ "Development Status :: 3 - Alpha",
21
+ ]
22
+ dependencies = ["numpy>=1.24"]
23
+
24
+ [project.urls]
25
+ Homepage = "https://github.com/satriabimantara/balinese-aisummarizer"
26
+
27
+ [project.optional-dependencies]
28
+ nlp = ["balinese-nlp==2.4.2"] # POS + SatuaNER features (needed by the pretrained models)
29
+ test = ["pytest>=8"]
30
+ train = ["metaheuristicpy>=3.2.0", "pandas", "openpyxl", "joblib"]
31
+
32
+ [tool.setuptools.packages.find]
33
+ include = ["satuasummarization*"]
34
+
35
+ [tool.setuptools.package-data]
36
+ "satuasummarization.extractive" = ["pretrained_optimal_models/*.json"]
37
+
38
+ [tool.pytest.ini_options]
39
+ testpaths = ["tests"]
@@ -0,0 +1,2 @@
1
+ """satuasummarization: Balinese text summarisation (extractive: metaheuristic feature weighting; abstractive: planned)."""
2
+ __version__ = "0.1.0"
@@ -0,0 +1 @@
1
+ """Abstractive (LLM-based) Balinese summarisation — reserved, not implemented yet."""
@@ -0,0 +1,14 @@
1
+ """Extractive Balinese summarisation with metaheuristic-optimised feature weights.
2
+
3
+ >>> from satuasummarization.extractive import MetaheuristicSummarizer
4
+ >>> model = MetaheuristicSummarizer.from_pretrained("best")
5
+ >>> model.summarize(text, title="I Kancil", ratio=0.3)["summary"]
6
+ """
7
+ from .data import Document, load_corpus, train_test_split_docs
8
+ from .features import FEATURE_NAMES, SentenceFeatureExtractor, split_sentences, tokenize
9
+ from .metrics import evaluate_summary, rouge_l, rouge_n
10
+ from .summarizer import MetaheuristicSummarizer, available_models
11
+
12
+ __all__ = ["Document", "FEATURE_NAMES", "MetaheuristicSummarizer", "SentenceFeatureExtractor", "available_models",
13
+ "evaluate_summary", "load_corpus", "rouge_l", "rouge_n", "split_sentences", "tokenize",
14
+ "train_test_split_docs"]
@@ -0,0 +1,52 @@
1
+ """Loader for the BaliSummarization extractive corpus (one .xlsx per document with the columns
2
+ ``sentences`` and ``extractive_summary``; the file name is the title)."""
3
+ from dataclasses import dataclass, field
4
+ from pathlib import Path
5
+
6
+ import numpy as np
7
+
8
+ from .features import tokenize
9
+
10
+
11
+ @dataclass
12
+ class Document:
13
+ title: str
14
+ sentences: list
15
+ labels: np.ndarray
16
+ source: str = ""
17
+ subset: str = ""
18
+ features: np.ndarray = field(default=None, repr=False)
19
+
20
+
21
+ def load_corpus(root, subsets=("complete", "independen", "pilot", "pre")):
22
+ """Read every document under ``root/<source>/<subset>/*.xlsx``. Rows without any word token
23
+ (stray quote marks) are dropped together with their label."""
24
+ import pandas as pd
25
+
26
+ docs = []
27
+ for path in sorted(Path(root).glob("*/*/*.xlsx")):
28
+ source, subset = path.parent.parent.name, path.parent.name
29
+ if subset not in subsets:
30
+ continue
31
+ df = pd.read_excel(path)
32
+ if not {"sentences", "extractive_summary"} <= set(df.columns):
33
+ raise ValueError(f"{path}: expected columns 'sentences' and 'extractive_summary'")
34
+ df = df[df["sentences"].astype(str).map(lambda s: bool(tokenize(s)))]
35
+ labels = df["extractive_summary"].fillna(0).astype(int).clip(0, 1).to_numpy()
36
+ if len(df) < 2 or labels.sum() == 0:
37
+ continue
38
+ docs.append(Document(path.stem, df["sentences"].astype(str).str.strip().tolist(), labels, source, subset))
39
+ return docs
40
+
41
+
42
+ def train_test_split_docs(docs, test_size=0.2, seed=42):
43
+ """Split documents (not sentences) with the same proportion inside every source."""
44
+ rng = np.random.default_rng(seed)
45
+ train, test = [], []
46
+ for source in sorted({d.source for d in docs}):
47
+ group = [d for d in docs if d.source == source]
48
+ idx = rng.permutation(len(group))
49
+ k = max(1, int(round(test_size * len(group))))
50
+ test += [group[i] for i in idx[:k]]
51
+ train += [group[i] for i in idx[k:]]
52
+ return train, test
@@ -0,0 +1,177 @@
1
+ """Sentence feature extraction for Balinese extractive summarisation.
2
+
3
+ Each sentence of a document is described by 13 features, min-max normalised inside the document so that
4
+ weights learned on one corpus transfer to documents of any length:
5
+
6
+ == ===================== =====================================================================
7
+ # name definition
8
+ == ===================== =====================================================================
9
+ 1 sentence_length tokens in the sentence / tokens in the longest sentence
10
+ 2 sentence_position 1 - index / n (earlier sentences score higher)
11
+ 3 numerical_ratio share of numeric tokens
12
+ 4 title_jaccard Jaccard overlap between sentence and title vocabularies
13
+ 5 tfisf_mean mean TF-ISF weight of the sentence tokens (ISF computed in the document)
14
+ 6 centroid_cosine cosine(TF-ISF sentence vector, TF-ISF document centroid)
15
+ 7 title_cosine cosine(TF-ISF sentence vector, TF-ISF title vector)
16
+ 8 intra_similarity mean cosine to the other sentences of the document
17
+ 9 dialogue_ratio share of tokens inside quotation marks (direct speech)
18
+ 10 noun_density NN/NND tags / tokens (balinese_nlp HMM POS tagger)
19
+ 11 verb_density VB/MD tags / tokens
20
+ 12 propernoun_density NNP tags / tokens
21
+ 13 character_density characters found by SatuaNER / tokens
22
+ == ===================== =====================================================================
23
+
24
+ Features 10-13 need ``balinese-nlp`` (``pip install satuasummarization[nlp]``); with
25
+ ``linguistic=False`` they are returned as zeros (the pretrained weights expect all 13).
26
+ """
27
+ import re
28
+
29
+ import numpy as np
30
+
31
+ FEATURE_NAMES = [
32
+ "sentence_length", "sentence_position", "numerical_ratio", "title_jaccard", "tfisf_mean",
33
+ "centroid_cosine", "title_cosine", "intra_similarity", "dialogue_ratio",
34
+ "noun_density", "verb_density", "propernoun_density", "character_density",
35
+ ]
36
+ _TOKEN = re.compile(r"[^\W_]+(?:[-'][^\W_]+)*", re.UNICODE)
37
+ _QUOTE = re.compile(r"(``|''|\"|“|”)")
38
+
39
+
40
+ def tokenize(text):
41
+ """Lower-cased word tokens (letters/digits, hyphenated words kept together)."""
42
+ return _TOKEN.findall(str(text).lower())
43
+
44
+
45
+ def split_sentences(text):
46
+ """Rule-based sentence splitter for raw Balinese text (., !, ? followed by space and a capital,
47
+ quote or digit; newlines always split)."""
48
+ parts = []
49
+ for block in re.split(r"\n\s*\n|\r?\n", str(text)):
50
+ block = block.strip()
51
+ if block:
52
+ parts.extend(s.strip() for s in re.split(r"(?<=[.!?])[\"'”’]*\s+(?=[\"'“‘(`]*[A-Z0-9])", block) if s.strip())
53
+ return [s for s in parts if tokenize(s)]
54
+
55
+
56
+ def _minmax(col):
57
+ lo, hi = col.min(), col.max()
58
+ return np.zeros_like(col) if hi - lo < 1e-12 else (col - lo) / (hi - lo)
59
+
60
+
61
+ class SentenceFeatureExtractor:
62
+ """Extract the 13 summarisation features of every sentence of one document.
63
+
64
+ >>> fx = SentenceFeatureExtractor()
65
+ >>> X = fx.transform(sentences, title="I Kancil") # shape (n_sentences, 13)
66
+ """
67
+
68
+ feature_names = FEATURE_NAMES
69
+
70
+ def __init__(self, linguistic=True, character_ner=True):
71
+ self.linguistic = linguistic
72
+ self.character_ner = character_ner and linguistic
73
+ self._pos = self._ner = None
74
+ if linguistic:
75
+ try:
76
+ from balinese_nlp.postag.HMM import HiddenMarkovModelPOSTag
77
+ except ImportError as exc:
78
+ raise ImportError("linguistic features need balinese-nlp: pip install satuasummarization[nlp] "
79
+ "or use SentenceFeatureExtractor(linguistic=False)") from exc
80
+ self._pos = HiddenMarkovModelPOSTag()
81
+ if self.character_ner:
82
+ from balinese_nlp.narratives.utils import load_pretrained_character_ner_model
83
+ self._ner = load_pretrained_character_ner_model(modelversion="satuaner")
84
+
85
+ # ------------------------------------------------------------------ helpers
86
+ @staticmethod
87
+ def _tfisf(token_lists, title_tokens):
88
+ vocab = sorted({t for toks in token_lists for t in toks} | set(title_tokens))
89
+ index = {t: i for i, t in enumerate(vocab)}
90
+ n = len(token_lists)
91
+ tf = np.zeros((n, len(vocab)))
92
+ for i, toks in enumerate(token_lists):
93
+ for t in toks:
94
+ tf[i, index[t]] += 1
95
+ df = (tf > 0).sum(0)
96
+ isf = np.log((n + 1) / (df + 1)) + 1
97
+ S = tf * isf
98
+ T = np.zeros(len(vocab))
99
+ for t in title_tokens:
100
+ T[index[t]] += 1
101
+ return S, T * isf
102
+
103
+ @staticmethod
104
+ def _cos(A, b):
105
+ na = np.linalg.norm(A, axis=1)
106
+ nb = np.linalg.norm(b)
107
+ out = np.zeros(len(A))
108
+ ok = (na > 0) & (nb > 0)
109
+ out[ok] = A[ok] @ b / (na[ok] * nb)
110
+ return out
111
+
112
+ def _linguistic(self, sentences, n_tokens):
113
+ out = np.zeros((len(sentences), 4))
114
+ self.linguistic_failures_ = 0
115
+ if not self.linguistic:
116
+ return out
117
+ for i, s in enumerate(sentences):
118
+ s = " ".join(s.split()) # repeated spaces make empty tokens that crash the HMM tagger
119
+ try:
120
+ tagged = self._pos.predict(s).replace("\n", " ").split()
121
+ tags = [tok.rsplit("/", 1)[-1] for tok in tagged if "/" in tok]
122
+ m = max(len(tags), 1)
123
+ out[i, 0] = sum(t in ("NN", "NND") for t in tags) / m
124
+ out[i, 1] = sum(t in ("VB", "MD") for t in tags) / m
125
+ out[i, 2] = sum(t == "NNP" for t in tags) / m
126
+ if self._ner is not None:
127
+ y, _ = self._ner.predict_sentence(s)
128
+ chars = self._ner.extract_predicted_characters_from_sentence(s, y)["predicted_chars"]
129
+ if isinstance(chars, str) and chars:
130
+ out[i, 3] = len(chars.split("; ")) / max(n_tokens[i], 1)
131
+ except (IndexError, KeyError, ValueError):
132
+ # shortcut: a sentence balinese-nlp cannot tag keeps 0 linguistic features — revisit when the
133
+ # tagger handles every token; counted in linguistic_failures_ so callers can see it
134
+ out[i] = 0
135
+ self.linguistic_failures_ += 1
136
+ return out
137
+
138
+ # ------------------------------------------------------------------ API
139
+ def transform(self, sentences, title=""):
140
+ """Return an (n_sentences, 13) array of document-normalised features."""
141
+ sentences = [str(s) for s in sentences]
142
+ if not sentences:
143
+ raise ValueError("sentences must not be empty")
144
+ toks = [tokenize(s) for s in sentences]
145
+ n = len(sentences)
146
+ n_tok = np.array([len(t) for t in toks], dtype=float)
147
+ title_tokens = tokenize(title)
148
+ title_set = set(title_tokens)
149
+ S, T = self._tfisf(toks, title_tokens)
150
+
151
+ F = np.zeros((n, len(FEATURE_NAMES)))
152
+ F[:, 0] = n_tok / max(n_tok.max(), 1)
153
+ F[:, 1] = 1 - np.arange(n) / n
154
+ F[:, 2] = [sum(t.isdigit() for t in tk) / max(len(tk), 1) for tk in toks]
155
+ F[:, 3] = [len(set(tk) & title_set) / len(set(tk) | title_set) if (tk or title_set) else 0 for tk in toks]
156
+ F[:, 4] = S.sum(1) / np.maximum(n_tok, 1)
157
+ F[:, 5] = self._cos(S, S.mean(0))
158
+ F[:, 6] = self._cos(S, T)
159
+ norms = np.linalg.norm(S, axis=1, keepdims=True)
160
+ U = np.divide(S, norms, out=np.zeros_like(S), where=norms > 0)
161
+ sim = U @ U.T
162
+ F[:, 7] = (sim.sum(1) - np.diag(sim)) / max(n - 1, 1)
163
+ F[:, 8] = [self._dialogue_ratio(s) for s in sentences]
164
+ F[:, 9:13] = self._linguistic(sentences, n_tok)
165
+ return np.column_stack([_minmax(F[:, j]) for j in range(F.shape[1])])
166
+
167
+ @staticmethod
168
+ def _dialogue_ratio(sentence):
169
+ inside, count, total = False, 0, 0
170
+ for piece in _QUOTE.split(sentence):
171
+ if _QUOTE.fullmatch(piece or " "):
172
+ inside = not inside
173
+ continue
174
+ k = len(tokenize(piece))
175
+ total += k
176
+ count += k if inside else 0
177
+ return count / total if total else 0.0