satuasummarization 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- satuasummarization/__init__.py +2 -0
- satuasummarization/abstractive/__init__.py +1 -0
- satuasummarization/extractive/__init__.py +14 -0
- satuasummarization/extractive/data.py +52 -0
- satuasummarization/extractive/features.py +177 -0
- satuasummarization/extractive/metrics.py +101 -0
- satuasummarization/extractive/pretrained_optimal_models/AHO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/AO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/AOA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/APO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/BEO.json +144 -0
- satuasummarization/extractive/pretrained_optimal_models/CFO.json +169 -0
- satuasummarization/extractive/pretrained_optimal_models/CFOA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/DIO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/DRA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/ECO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/EEGWO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/EGO.json +169 -0
- satuasummarization/extractive/pretrained_optimal_models/EPO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/FATA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/FLA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/GA.json +226 -0
- satuasummarization/extractive/pretrained_optimal_models/GCRA.json +188 -0
- satuasummarization/extractive/pretrained_optimal_models/GTO.json +169 -0
- satuasummarization/extractive/pretrained_optimal_models/GWO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/GWOWOA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/HGS.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/HGWODE.json +169 -0
- satuasummarization/extractive/pretrained_optimal_models/HHO.json +180 -0
- satuasummarization/extractive/pretrained_optimal_models/HO.json +152 -0
- satuasummarization/extractive/pretrained_optimal_models/HOA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/IAKMA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/IGWO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/INFO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/KMA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/MBKA.json +169 -0
- satuasummarization/extractive/pretrained_optimal_models/MGO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/MGWO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/MO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/PFA.json +127 -0
- satuasummarization/extractive/pretrained_optimal_models/PLO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/PO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/PSO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/PSOGWO.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/PUMA.json +215 -0
- satuasummarization/extractive/pretrained_optimal_models/RIME.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/RUN.json +178 -0
- satuasummarization/extractive/pretrained_optimal_models/SAR.json +168 -0
- satuasummarization/extractive/pretrained_optimal_models/SMA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/SSA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/WOA.json +218 -0
- satuasummarization/extractive/pretrained_optimal_models/WaOA.json +152 -0
- satuasummarization/extractive/pretrained_optimal_models/index.json +679 -0
- satuasummarization/extractive/pretrained_optimal_models/mEGO.json +169 -0
- satuasummarization/extractive/summarizer.py +163 -0
- satuasummarization-0.1.0.dist-info/METADATA +163 -0
- satuasummarization-0.1.0.dist-info/RECORD +60 -0
- satuasummarization-0.1.0.dist-info/WHEEL +5 -0
- satuasummarization-0.1.0.dist-info/licenses/LICENSE +28 -0
- satuasummarization-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Abstractive (LLM-based) Balinese summarisation — reserved, not implemented yet."""
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Extractive Balinese summarisation with metaheuristic-optimised feature weights.
|
|
2
|
+
|
|
3
|
+
>>> from satuasummarization.extractive import MetaheuristicSummarizer
|
|
4
|
+
>>> model = MetaheuristicSummarizer.from_pretrained("best")
|
|
5
|
+
>>> model.summarize(text, title="I Kancil", ratio=0.3)["summary"]
|
|
6
|
+
"""
|
|
7
|
+
from .data import Document, load_corpus, train_test_split_docs
|
|
8
|
+
from .features import FEATURE_NAMES, SentenceFeatureExtractor, split_sentences, tokenize
|
|
9
|
+
from .metrics import evaluate_summary, rouge_l, rouge_n
|
|
10
|
+
from .summarizer import MetaheuristicSummarizer, available_models
|
|
11
|
+
|
|
12
|
+
__all__ = ["Document", "FEATURE_NAMES", "MetaheuristicSummarizer", "SentenceFeatureExtractor", "available_models",
|
|
13
|
+
"evaluate_summary", "load_corpus", "rouge_l", "rouge_n", "split_sentences", "tokenize",
|
|
14
|
+
"train_test_split_docs"]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Loader for the BaliSummarization extractive corpus (one .xlsx per document with the columns
|
|
2
|
+
``sentences`` and ``extractive_summary``; the file name is the title)."""
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from .features import tokenize
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class Document:
|
|
13
|
+
title: str
|
|
14
|
+
sentences: list
|
|
15
|
+
labels: np.ndarray
|
|
16
|
+
source: str = ""
|
|
17
|
+
subset: str = ""
|
|
18
|
+
features: np.ndarray = field(default=None, repr=False)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def load_corpus(root, subsets=("complete", "independen", "pilot", "pre")):
|
|
22
|
+
"""Read every document under ``root/<source>/<subset>/*.xlsx``. Rows without any word token
|
|
23
|
+
(stray quote marks) are dropped together with their label."""
|
|
24
|
+
import pandas as pd
|
|
25
|
+
|
|
26
|
+
docs = []
|
|
27
|
+
for path in sorted(Path(root).glob("*/*/*.xlsx")):
|
|
28
|
+
source, subset = path.parent.parent.name, path.parent.name
|
|
29
|
+
if subset not in subsets:
|
|
30
|
+
continue
|
|
31
|
+
df = pd.read_excel(path)
|
|
32
|
+
if not {"sentences", "extractive_summary"} <= set(df.columns):
|
|
33
|
+
raise ValueError(f"{path}: expected columns 'sentences' and 'extractive_summary'")
|
|
34
|
+
df = df[df["sentences"].astype(str).map(lambda s: bool(tokenize(s)))]
|
|
35
|
+
labels = df["extractive_summary"].fillna(0).astype(int).clip(0, 1).to_numpy()
|
|
36
|
+
if len(df) < 2 or labels.sum() == 0:
|
|
37
|
+
continue
|
|
38
|
+
docs.append(Document(path.stem, df["sentences"].astype(str).str.strip().tolist(), labels, source, subset))
|
|
39
|
+
return docs
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def train_test_split_docs(docs, test_size=0.2, seed=42):
|
|
43
|
+
"""Split documents (not sentences) with the same proportion inside every source."""
|
|
44
|
+
rng = np.random.default_rng(seed)
|
|
45
|
+
train, test = [], []
|
|
46
|
+
for source in sorted({d.source for d in docs}):
|
|
47
|
+
group = [d for d in docs if d.source == source]
|
|
48
|
+
idx = rng.permutation(len(group))
|
|
49
|
+
k = max(1, int(round(test_size * len(group))))
|
|
50
|
+
test += [group[i] for i in idx[:k]]
|
|
51
|
+
train += [group[i] for i in idx[k:]]
|
|
52
|
+
return train, test
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"""Sentence feature extraction for Balinese extractive summarisation.
|
|
2
|
+
|
|
3
|
+
Each sentence of a document is described by 13 features, min-max normalised inside the document so that
|
|
4
|
+
weights learned on one corpus transfer to documents of any length:
|
|
5
|
+
|
|
6
|
+
== ===================== =====================================================================
|
|
7
|
+
# name definition
|
|
8
|
+
== ===================== =====================================================================
|
|
9
|
+
1 sentence_length tokens in the sentence / tokens in the longest sentence
|
|
10
|
+
2 sentence_position 1 - index / n (earlier sentences score higher)
|
|
11
|
+
3 numerical_ratio share of numeric tokens
|
|
12
|
+
4 title_jaccard Jaccard overlap between sentence and title vocabularies
|
|
13
|
+
5 tfisf_mean mean TF-ISF weight of the sentence tokens (ISF computed in the document)
|
|
14
|
+
6 centroid_cosine cosine(TF-ISF sentence vector, TF-ISF document centroid)
|
|
15
|
+
7 title_cosine cosine(TF-ISF sentence vector, TF-ISF title vector)
|
|
16
|
+
8 intra_similarity mean cosine to the other sentences of the document
|
|
17
|
+
9 dialogue_ratio share of tokens inside quotation marks (direct speech)
|
|
18
|
+
10 noun_density NN/NND tags / tokens (balinese_nlp HMM POS tagger)
|
|
19
|
+
11 verb_density VB/MD tags / tokens
|
|
20
|
+
12 propernoun_density NNP tags / tokens
|
|
21
|
+
13 character_density characters found by SatuaNER / tokens
|
|
22
|
+
== ===================== =====================================================================
|
|
23
|
+
|
|
24
|
+
Features 10-13 need ``balinese-nlp`` (``pip install satuasummarization[nlp]``); with
|
|
25
|
+
``linguistic=False`` they are returned as zeros (the pretrained weights expect all 13).
|
|
26
|
+
"""
|
|
27
|
+
import re
|
|
28
|
+
|
|
29
|
+
import numpy as np
|
|
30
|
+
|
|
31
|
+
FEATURE_NAMES = [
|
|
32
|
+
"sentence_length", "sentence_position", "numerical_ratio", "title_jaccard", "tfisf_mean",
|
|
33
|
+
"centroid_cosine", "title_cosine", "intra_similarity", "dialogue_ratio",
|
|
34
|
+
"noun_density", "verb_density", "propernoun_density", "character_density",
|
|
35
|
+
]
|
|
36
|
+
_TOKEN = re.compile(r"[^\W_]+(?:[-'][^\W_]+)*", re.UNICODE)
|
|
37
|
+
_QUOTE = re.compile(r"(``|''|\"|“|”)")
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def tokenize(text):
|
|
41
|
+
"""Lower-cased word tokens (letters/digits, hyphenated words kept together)."""
|
|
42
|
+
return _TOKEN.findall(str(text).lower())
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def split_sentences(text):
|
|
46
|
+
"""Rule-based sentence splitter for raw Balinese text (., !, ? followed by space and a capital,
|
|
47
|
+
quote or digit; newlines always split)."""
|
|
48
|
+
parts = []
|
|
49
|
+
for block in re.split(r"\n\s*\n|\r?\n", str(text)):
|
|
50
|
+
block = block.strip()
|
|
51
|
+
if block:
|
|
52
|
+
parts.extend(s.strip() for s in re.split(r"(?<=[.!?])[\"'”’]*\s+(?=[\"'“‘(`]*[A-Z0-9])", block) if s.strip())
|
|
53
|
+
return [s for s in parts if tokenize(s)]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _minmax(col):
|
|
57
|
+
lo, hi = col.min(), col.max()
|
|
58
|
+
return np.zeros_like(col) if hi - lo < 1e-12 else (col - lo) / (hi - lo)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SentenceFeatureExtractor:
|
|
62
|
+
"""Extract the 13 summarisation features of every sentence of one document.
|
|
63
|
+
|
|
64
|
+
>>> fx = SentenceFeatureExtractor()
|
|
65
|
+
>>> X = fx.transform(sentences, title="I Kancil") # shape (n_sentences, 13)
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
feature_names = FEATURE_NAMES
|
|
69
|
+
|
|
70
|
+
def __init__(self, linguistic=True, character_ner=True):
|
|
71
|
+
self.linguistic = linguistic
|
|
72
|
+
self.character_ner = character_ner and linguistic
|
|
73
|
+
self._pos = self._ner = None
|
|
74
|
+
if linguistic:
|
|
75
|
+
try:
|
|
76
|
+
from balinese_nlp.postag.HMM import HiddenMarkovModelPOSTag
|
|
77
|
+
except ImportError as exc:
|
|
78
|
+
raise ImportError("linguistic features need balinese-nlp: pip install satuasummarization[nlp] "
|
|
79
|
+
"or use SentenceFeatureExtractor(linguistic=False)") from exc
|
|
80
|
+
self._pos = HiddenMarkovModelPOSTag()
|
|
81
|
+
if self.character_ner:
|
|
82
|
+
from balinese_nlp.narratives.utils import load_pretrained_character_ner_model
|
|
83
|
+
self._ner = load_pretrained_character_ner_model(modelversion="satuaner")
|
|
84
|
+
|
|
85
|
+
# ------------------------------------------------------------------ helpers
|
|
86
|
+
@staticmethod
|
|
87
|
+
def _tfisf(token_lists, title_tokens):
|
|
88
|
+
vocab = sorted({t for toks in token_lists for t in toks} | set(title_tokens))
|
|
89
|
+
index = {t: i for i, t in enumerate(vocab)}
|
|
90
|
+
n = len(token_lists)
|
|
91
|
+
tf = np.zeros((n, len(vocab)))
|
|
92
|
+
for i, toks in enumerate(token_lists):
|
|
93
|
+
for t in toks:
|
|
94
|
+
tf[i, index[t]] += 1
|
|
95
|
+
df = (tf > 0).sum(0)
|
|
96
|
+
isf = np.log((n + 1) / (df + 1)) + 1
|
|
97
|
+
S = tf * isf
|
|
98
|
+
T = np.zeros(len(vocab))
|
|
99
|
+
for t in title_tokens:
|
|
100
|
+
T[index[t]] += 1
|
|
101
|
+
return S, T * isf
|
|
102
|
+
|
|
103
|
+
@staticmethod
|
|
104
|
+
def _cos(A, b):
|
|
105
|
+
na = np.linalg.norm(A, axis=1)
|
|
106
|
+
nb = np.linalg.norm(b)
|
|
107
|
+
out = np.zeros(len(A))
|
|
108
|
+
ok = (na > 0) & (nb > 0)
|
|
109
|
+
out[ok] = A[ok] @ b / (na[ok] * nb)
|
|
110
|
+
return out
|
|
111
|
+
|
|
112
|
+
def _linguistic(self, sentences, n_tokens):
|
|
113
|
+
out = np.zeros((len(sentences), 4))
|
|
114
|
+
self.linguistic_failures_ = 0
|
|
115
|
+
if not self.linguistic:
|
|
116
|
+
return out
|
|
117
|
+
for i, s in enumerate(sentences):
|
|
118
|
+
s = " ".join(s.split()) # repeated spaces make empty tokens that crash the HMM tagger
|
|
119
|
+
try:
|
|
120
|
+
tagged = self._pos.predict(s).replace("\n", " ").split()
|
|
121
|
+
tags = [tok.rsplit("/", 1)[-1] for tok in tagged if "/" in tok]
|
|
122
|
+
m = max(len(tags), 1)
|
|
123
|
+
out[i, 0] = sum(t in ("NN", "NND") for t in tags) / m
|
|
124
|
+
out[i, 1] = sum(t in ("VB", "MD") for t in tags) / m
|
|
125
|
+
out[i, 2] = sum(t == "NNP" for t in tags) / m
|
|
126
|
+
if self._ner is not None:
|
|
127
|
+
y, _ = self._ner.predict_sentence(s)
|
|
128
|
+
chars = self._ner.extract_predicted_characters_from_sentence(s, y)["predicted_chars"]
|
|
129
|
+
if isinstance(chars, str) and chars:
|
|
130
|
+
out[i, 3] = len(chars.split("; ")) / max(n_tokens[i], 1)
|
|
131
|
+
except (IndexError, KeyError, ValueError):
|
|
132
|
+
# shortcut: a sentence balinese-nlp cannot tag keeps 0 linguistic features — revisit when the
|
|
133
|
+
# tagger handles every token; counted in linguistic_failures_ so callers can see it
|
|
134
|
+
out[i] = 0
|
|
135
|
+
self.linguistic_failures_ += 1
|
|
136
|
+
return out
|
|
137
|
+
|
|
138
|
+
# ------------------------------------------------------------------ API
|
|
139
|
+
def transform(self, sentences, title=""):
|
|
140
|
+
"""Return an (n_sentences, 13) array of document-normalised features."""
|
|
141
|
+
sentences = [str(s) for s in sentences]
|
|
142
|
+
if not sentences:
|
|
143
|
+
raise ValueError("sentences must not be empty")
|
|
144
|
+
toks = [tokenize(s) for s in sentences]
|
|
145
|
+
n = len(sentences)
|
|
146
|
+
n_tok = np.array([len(t) for t in toks], dtype=float)
|
|
147
|
+
title_tokens = tokenize(title)
|
|
148
|
+
title_set = set(title_tokens)
|
|
149
|
+
S, T = self._tfisf(toks, title_tokens)
|
|
150
|
+
|
|
151
|
+
F = np.zeros((n, len(FEATURE_NAMES)))
|
|
152
|
+
F[:, 0] = n_tok / max(n_tok.max(), 1)
|
|
153
|
+
F[:, 1] = 1 - np.arange(n) / n
|
|
154
|
+
F[:, 2] = [sum(t.isdigit() for t in tk) / max(len(tk), 1) for tk in toks]
|
|
155
|
+
F[:, 3] = [len(set(tk) & title_set) / len(set(tk) | title_set) if (tk or title_set) else 0 for tk in toks]
|
|
156
|
+
F[:, 4] = S.sum(1) / np.maximum(n_tok, 1)
|
|
157
|
+
F[:, 5] = self._cos(S, S.mean(0))
|
|
158
|
+
F[:, 6] = self._cos(S, T)
|
|
159
|
+
norms = np.linalg.norm(S, axis=1, keepdims=True)
|
|
160
|
+
U = np.divide(S, norms, out=np.zeros_like(S), where=norms > 0)
|
|
161
|
+
sim = U @ U.T
|
|
162
|
+
F[:, 7] = (sim.sum(1) - np.diag(sim)) / max(n - 1, 1)
|
|
163
|
+
F[:, 8] = [self._dialogue_ratio(s) for s in sentences]
|
|
164
|
+
F[:, 9:13] = self._linguistic(sentences, n_tok)
|
|
165
|
+
return np.column_stack([_minmax(F[:, j]) for j in range(F.shape[1])])
|
|
166
|
+
|
|
167
|
+
@staticmethod
|
|
168
|
+
def _dialogue_ratio(sentence):
|
|
169
|
+
inside, count, total = False, 0, 0
|
|
170
|
+
for piece in _QUOTE.split(sentence):
|
|
171
|
+
if _QUOTE.fullmatch(piece or " "):
|
|
172
|
+
inside = not inside
|
|
173
|
+
continue
|
|
174
|
+
k = len(tokenize(piece))
|
|
175
|
+
total += k
|
|
176
|
+
count += k if inside else 0
|
|
177
|
+
return count / total if total else 0.0
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""ROUGE-N / ROUGE-L and sentence-level metrics for extractive summaries (lower-cased tokens, no stemming)."""
|
|
2
|
+
from collections import Counter
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from .features import tokenize
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _ngrams(tokens, n):
|
|
10
|
+
return Counter(tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _prf(overlap, n_sys, n_ref):
|
|
14
|
+
p = overlap / n_sys if n_sys else 0.0
|
|
15
|
+
r = overlap / n_ref if n_ref else 0.0
|
|
16
|
+
return p, r, (2 * p * r / (p + r) if p + r else 0.0)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def rouge_n(system, reference, n=1):
|
|
20
|
+
"""(precision, recall, f1) of ROUGE-N between two texts."""
|
|
21
|
+
s, r = _ngrams(tokenize(system), n), _ngrams(tokenize(reference), n)
|
|
22
|
+
return _prf(sum((s & r).values()), sum(s.values()), sum(r.values()))
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def rouge_l(system, reference):
|
|
26
|
+
"""(precision, recall, f1) of ROUGE-L (summary-level LCS on the concatenated texts)."""
|
|
27
|
+
a, b = tokenize(system), tokenize(reference)
|
|
28
|
+
if not a or not b:
|
|
29
|
+
return 0.0, 0.0, 0.0
|
|
30
|
+
prev = [0] * (len(b) + 1)
|
|
31
|
+
for x in a:
|
|
32
|
+
cur = [0]
|
|
33
|
+
for j, y in enumerate(b):
|
|
34
|
+
cur.append(prev[j] + 1 if x == y else max(prev[j + 1], cur[j]))
|
|
35
|
+
prev = cur
|
|
36
|
+
return _prf(prev[-1], len(a), len(b))
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def sentence_scores(y_true, y_pred):
|
|
40
|
+
"""Accuracy and macro precision/recall/F1 of the selected-sentence labels."""
|
|
41
|
+
y_true, y_pred = np.asarray(y_true, int), np.asarray(y_pred, int)
|
|
42
|
+
out = {"accuracy": float((y_true == y_pred).mean())}
|
|
43
|
+
ps, rs, fs = [], [], []
|
|
44
|
+
for c in (0, 1):
|
|
45
|
+
tp = np.sum((y_pred == c) & (y_true == c))
|
|
46
|
+
p, r, f = _prf(tp, np.sum(y_pred == c), np.sum(y_true == c))
|
|
47
|
+
ps.append(p), rs.append(r), fs.append(f)
|
|
48
|
+
out.update(precision_macro=float(np.mean(ps)), recall_macro=float(np.mean(rs)), f1_macro=float(np.mean(fs)))
|
|
49
|
+
return out
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _rouge_sentences(sys_sents, ref_sents, n):
|
|
53
|
+
"""ROUGE-N with n-grams counted inside sentences (no n-gram spans a sentence boundary)."""
|
|
54
|
+
s = sum((_ngrams(tokenize(x), n) for x in sys_sents), Counter())
|
|
55
|
+
r = sum((_ngrams(tokenize(x), n) for x in ref_sents), Counter())
|
|
56
|
+
return _prf(sum((s & r).values()), sum(s.values()), sum(r.values()))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def evaluate_summary(sentences, y_true, y_pred):
|
|
60
|
+
"""All metrics of one document: ROUGE-1/2/L (P/R/F1) between selected and reference sentences."""
|
|
61
|
+
sys_s = [s for s, k in zip(sentences, y_pred) if k]
|
|
62
|
+
ref_s = [s for s, k in zip(sentences, y_true) if k]
|
|
63
|
+
out = {}
|
|
64
|
+
for key, (p, r, f) in {"rouge1": _rouge_sentences(sys_s, ref_s, 1), "rouge2": _rouge_sentences(sys_s, ref_s, 2),
|
|
65
|
+
"rougeL": rouge_l(" ".join(sys_s), " ".join(ref_s))}.items():
|
|
66
|
+
out.update({f"{key}_precision": p, f"{key}_recall": r, f"{key}_f1": f})
|
|
67
|
+
out.update(sentence_scores(y_true, y_pred))
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class FastRouge:
|
|
72
|
+
"""Vectorised ROUGE-1/2 F1 of many candidate selections of one document (used as fitness).
|
|
73
|
+
|
|
74
|
+
Precomputes the sentence x n-gram count matrices once; ``f1(masks)`` scores a (k, n_sentences)
|
|
75
|
+
boolean matrix of selections against the reference labels in a few matrix operations."""
|
|
76
|
+
|
|
77
|
+
def __init__(self, sentences, labels):
|
|
78
|
+
toks = [tokenize(s) for s in sentences]
|
|
79
|
+
self.mats, self.refs = [], []
|
|
80
|
+
ref = np.asarray(labels, bool)
|
|
81
|
+
for n in (1, 2):
|
|
82
|
+
grams = [_ngrams(t, n) for t in toks]
|
|
83
|
+
vocab = {g: i for i, g in enumerate({g for c in grams for g in c})}
|
|
84
|
+
M = np.zeros((len(toks), max(len(vocab), 1)))
|
|
85
|
+
for i, c in enumerate(grams):
|
|
86
|
+
for g, k in c.items():
|
|
87
|
+
M[i, vocab[g]] = k
|
|
88
|
+
self.mats.append(M)
|
|
89
|
+
self.refs.append(M[ref].sum(0))
|
|
90
|
+
|
|
91
|
+
def f1(self, masks):
|
|
92
|
+
masks = np.atleast_2d(masks).astype(float)
|
|
93
|
+
out = []
|
|
94
|
+
for M, r in zip(self.mats, self.refs):
|
|
95
|
+
sel = masks @ M
|
|
96
|
+
overlap = np.minimum(sel, r).sum(1)
|
|
97
|
+
n_sys, n_ref = sel.sum(1), r.sum()
|
|
98
|
+
p = np.divide(overlap, n_sys, out=np.zeros_like(overlap), where=n_sys > 0)
|
|
99
|
+
rc = overlap / n_ref if n_ref else np.zeros_like(overlap)
|
|
100
|
+
out.append(np.divide(2 * p * rc, p + rc, out=np.zeros_like(p), where=(p + rc) > 0))
|
|
101
|
+
return out # [rouge1_f1, rouge2_f1], each of shape (k,)
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
{
|
|
2
|
+
"format_version": 1,
|
|
3
|
+
"algorithm": "AHO",
|
|
4
|
+
"ratio": 0.4,
|
|
5
|
+
"feature_names": [
|
|
6
|
+
"sentence_length",
|
|
7
|
+
"sentence_position",
|
|
8
|
+
"numerical_ratio",
|
|
9
|
+
"title_jaccard",
|
|
10
|
+
"tfisf_mean",
|
|
11
|
+
"centroid_cosine",
|
|
12
|
+
"title_cosine",
|
|
13
|
+
"intra_similarity",
|
|
14
|
+
"dialogue_ratio",
|
|
15
|
+
"noun_density",
|
|
16
|
+
"verb_density",
|
|
17
|
+
"propernoun_density",
|
|
18
|
+
"character_density"
|
|
19
|
+
],
|
|
20
|
+
"weights": [
|
|
21
|
+
0.9771890314,
|
|
22
|
+
0.6049857527,
|
|
23
|
+
0.0654283591,
|
|
24
|
+
0.5653806336,
|
|
25
|
+
0.5339307778,
|
|
26
|
+
0.5171570937,
|
|
27
|
+
0.2858113962,
|
|
28
|
+
0.6824346881,
|
|
29
|
+
0.082246073,
|
|
30
|
+
0.4111390565,
|
|
31
|
+
0.5079619758,
|
|
32
|
+
0.2956964139,
|
|
33
|
+
0.1264237999
|
|
34
|
+
],
|
|
35
|
+
"meta": {
|
|
36
|
+
"train_fitness": 0.6815663670817179,
|
|
37
|
+
"history": [
|
|
38
|
+
0.6771105843652399,
|
|
39
|
+
0.6778483316074008,
|
|
40
|
+
0.6782982045696839,
|
|
41
|
+
0.6799616029296794,
|
|
42
|
+
0.6799616029296794,
|
|
43
|
+
0.6799616029296794,
|
|
44
|
+
0.6808398018306605,
|
|
45
|
+
0.6808398018306605,
|
|
46
|
+
0.6813440015824628,
|
|
47
|
+
0.6813765612573995,
|
|
48
|
+
0.6814634845402953,
|
|
49
|
+
0.6815470138978551,
|
|
50
|
+
0.6815663670817179,
|
|
51
|
+
0.6815663670817179,
|
|
52
|
+
0.6815663670817179,
|
|
53
|
+
0.6815663670817179,
|
|
54
|
+
0.6815663670817179,
|
|
55
|
+
0.6815663670817179,
|
|
56
|
+
0.6815663670817179,
|
|
57
|
+
0.6815663670817179,
|
|
58
|
+
0.6815663670817179,
|
|
59
|
+
0.6815663670817179,
|
|
60
|
+
0.6815663670817179,
|
|
61
|
+
0.6815663670817179,
|
|
62
|
+
0.6815663670817179,
|
|
63
|
+
0.6815663670817179,
|
|
64
|
+
0.6815663670817179,
|
|
65
|
+
0.6815663670817179,
|
|
66
|
+
0.6815663670817179,
|
|
67
|
+
0.6815663670817179,
|
|
68
|
+
0.6815663670817179,
|
|
69
|
+
0.6815663670817179,
|
|
70
|
+
0.6815663670817179,
|
|
71
|
+
0.6815663670817179,
|
|
72
|
+
0.6815663670817179,
|
|
73
|
+
0.6815663670817179,
|
|
74
|
+
0.6815663670817179,
|
|
75
|
+
0.6815663670817179,
|
|
76
|
+
0.6815663670817179,
|
|
77
|
+
0.6815663670817179,
|
|
78
|
+
0.6815663670817179,
|
|
79
|
+
0.6815663670817179,
|
|
80
|
+
0.6815663670817179,
|
|
81
|
+
0.6815663670817179,
|
|
82
|
+
0.6815663670817179,
|
|
83
|
+
0.6815663670817179,
|
|
84
|
+
0.6815663670817179,
|
|
85
|
+
0.6815663670817179,
|
|
86
|
+
0.6815663670817179,
|
|
87
|
+
0.6815663670817179,
|
|
88
|
+
0.6815663670817179,
|
|
89
|
+
0.6815663670817179,
|
|
90
|
+
0.6815663670817179,
|
|
91
|
+
0.6815663670817179,
|
|
92
|
+
0.6815663670817179,
|
|
93
|
+
0.6815663670817179,
|
|
94
|
+
0.6815663670817179,
|
|
95
|
+
0.6815663670817179,
|
|
96
|
+
0.6815663670817179,
|
|
97
|
+
0.6815663670817179,
|
|
98
|
+
0.6815663670817179,
|
|
99
|
+
0.6815663670817179,
|
|
100
|
+
0.6815663670817179,
|
|
101
|
+
0.6815663670817179,
|
|
102
|
+
0.6815663670817179,
|
|
103
|
+
0.6815663670817179,
|
|
104
|
+
0.6815663670817179,
|
|
105
|
+
0.6815663670817179,
|
|
106
|
+
0.6815663670817179,
|
|
107
|
+
0.6815663670817179,
|
|
108
|
+
0.6815663670817179,
|
|
109
|
+
0.6815663670817179,
|
|
110
|
+
0.6815663670817179,
|
|
111
|
+
0.6815663670817179,
|
|
112
|
+
0.6815663670817179,
|
|
113
|
+
0.6815663670817179,
|
|
114
|
+
0.6815663670817179,
|
|
115
|
+
0.6815663670817179,
|
|
116
|
+
0.6815663670817179,
|
|
117
|
+
0.6815663670817179,
|
|
118
|
+
0.6815663670817179,
|
|
119
|
+
0.6815663670817179,
|
|
120
|
+
0.6815663670817179,
|
|
121
|
+
0.6815663670817179,
|
|
122
|
+
0.6815663670817179,
|
|
123
|
+
0.6815663670817179,
|
|
124
|
+
0.6815663670817179,
|
|
125
|
+
0.6815663670817179,
|
|
126
|
+
0.6815663670817179,
|
|
127
|
+
0.6815663670817179,
|
|
128
|
+
0.6815663670817179,
|
|
129
|
+
0.6815663670817179,
|
|
130
|
+
0.6815663670817179,
|
|
131
|
+
0.6815663670817179,
|
|
132
|
+
0.6815663670817179,
|
|
133
|
+
0.6815663670817179,
|
|
134
|
+
0.6815663670817179,
|
|
135
|
+
0.6815663670817179,
|
|
136
|
+
0.6815663670817179
|
|
137
|
+
],
|
|
138
|
+
"n_iter": 99,
|
|
139
|
+
"fes": 3000,
|
|
140
|
+
"train_time_s": 21.77,
|
|
141
|
+
"n_agents": 30,
|
|
142
|
+
"max_iter": 100,
|
|
143
|
+
"seed": 1,
|
|
144
|
+
"cv_mean": {
|
|
145
|
+
"rouge1_precision": 0.6603092344093611,
|
|
146
|
+
"rouge1_recall": 0.7613519848046376,
|
|
147
|
+
"rouge1_f1": 0.7039364287836011,
|
|
148
|
+
"rouge2_precision": 0.5946814905115488,
|
|
149
|
+
"rouge2_recall": 0.6862068992630627,
|
|
150
|
+
"rouge2_f1": 0.6342854870402238,
|
|
151
|
+
"rougeL_precision": 0.6029606839187597,
|
|
152
|
+
"rougeL_recall": 0.6922958049147103,
|
|
153
|
+
"rougeL_f1": 0.64168714238523,
|
|
154
|
+
"accuracy": 0.6556776068966856,
|
|
155
|
+
"precision_macro": 0.6234901263176497,
|
|
156
|
+
"recall_macro": 0.6234901263176497,
|
|
157
|
+
"f1_macro": 0.6234901263176497
|
|
158
|
+
},
|
|
159
|
+
"cv_std": {
|
|
160
|
+
"rouge1_precision": 0.01233233625741901,
|
|
161
|
+
"rouge1_recall": 0.006705426338989149,
|
|
162
|
+
"rouge1_f1": 0.00790736837447157,
|
|
163
|
+
"rouge2_precision": 0.010698957531457288,
|
|
164
|
+
"rouge2_recall": 0.007713869507707053,
|
|
165
|
+
"rouge2_f1": 0.007443490328496597,
|
|
166
|
+
"rougeL_precision": 0.010338572860344184,
|
|
167
|
+
"rougeL_recall": 0.00947172160094064,
|
|
168
|
+
"rougeL_f1": 0.007917888892386841,
|
|
169
|
+
"accuracy": 0.014798947726295753,
|
|
170
|
+
"precision_macro": 0.012043897534437903,
|
|
171
|
+
"recall_macro": 0.012043897534437903,
|
|
172
|
+
"f1_macro": 0.012043897534437912
|
|
173
|
+
},
|
|
174
|
+
"test_oracle_length": {
|
|
175
|
+
"rouge1_precision": 0.6887356502750042,
|
|
176
|
+
"rouge1_recall": 0.7818995353126844,
|
|
177
|
+
"rouge1_f1": 0.7289357802130692,
|
|
178
|
+
"rouge2_precision": 0.6289930091306007,
|
|
179
|
+
"rouge2_recall": 0.7156287899462286,
|
|
180
|
+
"rouge2_f1": 0.6660218675849859,
|
|
181
|
+
"rougeL_precision": 0.6329397562291325,
|
|
182
|
+
"rougeL_recall": 0.7158401230128865,
|
|
183
|
+
"rougeL_f1": 0.6686865418729655,
|
|
184
|
+
"accuracy": 0.6367214149202084,
|
|
185
|
+
"precision_macro": 0.6145061017684398,
|
|
186
|
+
"recall_macro": 0.6145061017684398,
|
|
187
|
+
"f1_macro": 0.6145061017684398
|
|
188
|
+
},
|
|
189
|
+
"test_ratio_0_4": {
|
|
190
|
+
"rouge1_precision": 0.680402348139292,
|
|
191
|
+
"rouge1_recall": 0.7688025097252205,
|
|
192
|
+
"rouge1_f1": 0.7113797128214486,
|
|
193
|
+
"rouge2_precision": 0.6185335127844271,
|
|
194
|
+
"rouge2_recall": 0.7014186415741925,
|
|
195
|
+
"rouge2_f1": 0.6476424197714917,
|
|
196
|
+
"rougeL_precision": 0.6243996370000586,
|
|
197
|
+
"rougeL_recall": 0.7029192908339313,
|
|
198
|
+
"rougeL_f1": 0.6515772227331247,
|
|
199
|
+
"accuracy": 0.6166379204131875,
|
|
200
|
+
"precision_macro": 0.6088961672975579,
|
|
201
|
+
"recall_macro": 0.6139557410840671,
|
|
202
|
+
"f1_macro": 0.6042281127790115
|
|
203
|
+
},
|
|
204
|
+
"train_fitness_seeds": [
|
|
205
|
+
0.6762875917665878,
|
|
206
|
+
0.6815663670817179,
|
|
207
|
+
0.6746830710206154
|
|
208
|
+
],
|
|
209
|
+
"split": {
|
|
210
|
+
"test_size": 0.2,
|
|
211
|
+
"seed": 42,
|
|
212
|
+
"n_train": 370,
|
|
213
|
+
"n_test": 93,
|
|
214
|
+
"cv_folds": 5,
|
|
215
|
+
"cv_seed": 42
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|