dobermann 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dobermann/__init__.py +18 -0
- dobermann/document.py +20 -0
- dobermann/evaluators/__init__.py +3 -0
- dobermann/evaluators/segmentation_evaluator.py +207 -0
- dobermann/segmenters/__init__.py +9 -0
- dobermann/segmenters/abstract.py +79 -0
- dobermann/segmenters/graphseg_embeddings.py +226 -0
- dobermann/segmenters/texttiling_embeddings.py +133 -0
- dobermann-0.1.0.dist-info/METADATA +82 -0
- dobermann-0.1.0.dist-info/RECORD +12 -0
- dobermann-0.1.0.dist-info/WHEEL +4 -0
- dobermann-0.1.0.dist-info/entry_points.txt +3 -0
dobermann/__init__.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# Public facing API
|
|
2
|
+
from .document import Document
|
|
3
|
+
from .evaluators import EvaluationResult, SegmentationEvaluator
|
|
4
|
+
from .segmenters import (
|
|
5
|
+
GraphSegEmbeddings,
|
|
6
|
+
SegmentationResult,
|
|
7
|
+
TextTilingEmbeddings,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
__all__ = [
|
|
11
|
+
"Document",
|
|
12
|
+
"TextTiling",
|
|
13
|
+
"TextTilingEmbeddings",
|
|
14
|
+
"GraphSegEmbeddings",
|
|
15
|
+
"SegmentationEvaluator",
|
|
16
|
+
"EvaluationResult",
|
|
17
|
+
"SegmentationResult",
|
|
18
|
+
]
|
dobermann/document.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# TODO: implement document import
|
|
2
|
+
# create document from dataset
|
|
3
|
+
# create document from raw text, document from text file.
|
|
4
|
+
# create document from HTML
|
|
5
|
+
# create document from URL
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
import nltk
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Document:
|
|
14
|
+
sentences: list[str]
|
|
15
|
+
|
|
16
|
+
@classmethod
|
|
17
|
+
def from_text(cls, text: str) -> "Document":
|
|
18
|
+
sentences = nltk.sent_tokenize(text)
|
|
19
|
+
|
|
20
|
+
return cls(sentences=sentences)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
from nltk.metrics import ghd as ghd
|
|
6
|
+
from nltk.metrics import pk as pk
|
|
7
|
+
from nltk.metrics import windowdiff as wd
|
|
8
|
+
|
|
9
|
+
# NOTE: only set k window size if segment lengths are highly variable
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass(slots=True, frozen=True)
|
|
13
|
+
class EvaluationResult:
|
|
14
|
+
"""Segmentation evluation result container.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
hyp_str: Hypothesized segmentation string representation.
|
|
18
|
+
ref_str: Reference (ground truth) segmentation string representation.
|
|
19
|
+
pk: Pk error metrics under different settings.
|
|
20
|
+
wd: WindowDiff error metrics under different settings.
|
|
21
|
+
ghd: Generalized Hamming Distance error metric
|
|
22
|
+
runtime: Evaluation process execution time in seconds.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
hyp_str: str
|
|
26
|
+
ref_str: str
|
|
27
|
+
pk: dict[str, float]
|
|
28
|
+
wd: dict[str, float]
|
|
29
|
+
ghd: float
|
|
30
|
+
runtime: float
|
|
31
|
+
|
|
32
|
+
def __str__(self) -> str:
|
|
33
|
+
"""Evaluation result container summary tostring function.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
str: Multi-line human-readable and interpretable summary of evaluation result.
|
|
37
|
+
"""
|
|
38
|
+
width = 34
|
|
39
|
+
|
|
40
|
+
lines = [
|
|
41
|
+
"EvaluationResult",
|
|
42
|
+
"-" * width,
|
|
43
|
+
f"{'Metric':<20}{'Value':>14}",
|
|
44
|
+
"-" * width,
|
|
45
|
+
f"{'Pk (small)':<20}{self.pk['small']:>14.4f}",
|
|
46
|
+
f"{'Pk (default)':<20}{self.pk['default']:>14.4f}",
|
|
47
|
+
f"{'Pk (large)':<20}{self.pk['large']:>14.4f}",
|
|
48
|
+
f"{'Pk (nltk)':<20}{self.pk['nltk']:>14.4f}",
|
|
49
|
+
"",
|
|
50
|
+
f"{'WD (small)':<20}{self.wd['small']:>14.4f}",
|
|
51
|
+
f"{'WD (default)':<20}{self.wd['default']:>14.4f}",
|
|
52
|
+
f"{'WD (large)':<20}{self.wd['large']:>14.4f}",
|
|
53
|
+
"",
|
|
54
|
+
f"{'GHD':<20}{self.ghd:>14.4f}",
|
|
55
|
+
f"{'Runtime (s)':<20}{self.runtime:>14.4f}",
|
|
56
|
+
"-" * width,
|
|
57
|
+
f"REF {self.ref_str}",
|
|
58
|
+
f"HYP {self.hyp_str}",
|
|
59
|
+
]
|
|
60
|
+
return "\n".join(lines)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class SegmentationEvaluator:
|
|
64
|
+
"""Stateful evaluator for comparing segmentation results against ground truth.
|
|
65
|
+
|
|
66
|
+
Attributes:
|
|
67
|
+
ins_cost: Cost of inserting a boundary.
|
|
68
|
+
del_cost: Cost of deleting a boundary.
|
|
69
|
+
shift_cost_coeff: Cost of shifting a boundary.
|
|
70
|
+
boundary_symbol: Character symbol that denotes a boundary.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
def __init__(
|
|
74
|
+
self,
|
|
75
|
+
ins_cost=2.0,
|
|
76
|
+
del_cost=2.0,
|
|
77
|
+
shift_cost_coeff=1.0,
|
|
78
|
+
boundary_symbol="1",
|
|
79
|
+
):
|
|
80
|
+
|
|
81
|
+
self.ins_cost = ins_cost
|
|
82
|
+
self.del_cost = del_cost
|
|
83
|
+
self.shift_cost_coeff = shift_cost_coeff
|
|
84
|
+
self.boundary_symbol = boundary_symbol
|
|
85
|
+
|
|
86
|
+
def evaluate(
|
|
87
|
+
self,
|
|
88
|
+
ref_len: list[int],
|
|
89
|
+
hyp_len: list[int],
|
|
90
|
+
) -> EvaluationResult:
|
|
91
|
+
"""Evaluation function
|
|
92
|
+
|
|
93
|
+
Args:
|
|
94
|
+
ref_len: Reference (ground truth) segment lengths.
|
|
95
|
+
hyp_len: Hypothesized segment lengths.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
EvaluationResult: Evaluation result container.
|
|
99
|
+
"""
|
|
100
|
+
start_time = time.perf_counter()
|
|
101
|
+
|
|
102
|
+
self._validate_input(ref_len, hyp_len)
|
|
103
|
+
|
|
104
|
+
# string boundary representation
|
|
105
|
+
ref_str = self._lengths_to_str(ref_len)
|
|
106
|
+
hyp_str = self._lengths_to_str(hyp_len)
|
|
107
|
+
|
|
108
|
+
self._validate_conversion(str_rep=ref_str, len_rep=ref_len)
|
|
109
|
+
self._validate_conversion(str_rep=hyp_str, len_rep=hyp_len)
|
|
110
|
+
|
|
111
|
+
ghd_score = ghd(
|
|
112
|
+
ref_str,
|
|
113
|
+
hyp_str,
|
|
114
|
+
self.ins_cost,
|
|
115
|
+
self.del_cost,
|
|
116
|
+
self.shift_cost_coeff,
|
|
117
|
+
boundary=self.boundary_symbol,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
pk_scores = {}
|
|
121
|
+
wd_scores = {}
|
|
122
|
+
|
|
123
|
+
k_values = self._k_values(ref_str, ref_len)
|
|
124
|
+
for k_name, k in k_values.items():
|
|
125
|
+
pk_scores[k_name] = pk(ref_str, hyp_str, k=k, boundary=self.boundary_symbol)
|
|
126
|
+
wd_scores[k_name] = wd(ref_str, hyp_str, k=k, boundary=self.boundary_symbol)
|
|
127
|
+
|
|
128
|
+
# optional: include nltk default (k=None)
|
|
129
|
+
pk_scores["nltk"] = pk(ref_str, hyp_str, k=None, boundary=self.boundary_symbol)
|
|
130
|
+
|
|
131
|
+
end_time = time.perf_counter()
|
|
132
|
+
runtime = end_time - start_time
|
|
133
|
+
|
|
134
|
+
return EvaluationResult(
|
|
135
|
+
hyp_str=hyp_str,
|
|
136
|
+
ref_str=ref_str,
|
|
137
|
+
pk=pk_scores,
|
|
138
|
+
wd=wd_scores,
|
|
139
|
+
ghd=ghd_score,
|
|
140
|
+
runtime=runtime,
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
def _lengths_to_str(self, lengths: list[int]) -> str:
|
|
144
|
+
s = []
|
|
145
|
+
for i, length in enumerate(lengths):
|
|
146
|
+
s.extend(["0"] * (length - 1))
|
|
147
|
+
if i != len(lengths) - 1:
|
|
148
|
+
s.append("1")
|
|
149
|
+
return "".join(s)
|
|
150
|
+
|
|
151
|
+
def _k_values(self, ref_str, ref_lengths: list[int]) -> dict[str, int]:
|
|
152
|
+
|
|
153
|
+
# percentile approach
|
|
154
|
+
k_small = max(1, int(round(np.percentile(ref_lengths, 25) / 2)))
|
|
155
|
+
# k_default = max(1, int(round(np.mean(ref_lengths) / 2)))
|
|
156
|
+
k_default = self._nltk_default_k(ref_str)
|
|
157
|
+
k_large = max(1, int(round(np.percentile(ref_lengths, 75) / 2)))
|
|
158
|
+
|
|
159
|
+
# min and max approach
|
|
160
|
+
# k_small = max(1, int(round(min(ref_lengths) / 2)))
|
|
161
|
+
# k_default = int(round((sum(ref_lengths) / len(ref_lenghts)) / 2))
|
|
162
|
+
# k_large = int(round(max(ref_lengths) / 2))
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
"small": k_small,
|
|
166
|
+
"default": k_default,
|
|
167
|
+
"large": k_large,
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
def _validate_input(self, ref_len, hyp_len):
|
|
171
|
+
# TODO: add more meaningful error messages
|
|
172
|
+
ref_sum = sum(ref_len)
|
|
173
|
+
hyp_sum = sum(hyp_len)
|
|
174
|
+
|
|
175
|
+
if ref_sum != hyp_sum:
|
|
176
|
+
raise ValueError(
|
|
177
|
+
f"Segmentations must cover the same total length "
|
|
178
|
+
f"(number of sentences).\n"
|
|
179
|
+
f"Got: sum(ref_len)={ref_sum}, sum(hyp_len)={hyp_sum}"
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
invalid_ref = [len for len in ref_len if len <= 0]
|
|
183
|
+
invalid_hyp = [len for len in hyp_len if len <= 0]
|
|
184
|
+
|
|
185
|
+
if invalid_ref or invalid_hyp:
|
|
186
|
+
raise ValueError(
|
|
187
|
+
"Segment lengths must all be positive integers.\n"
|
|
188
|
+
f"Invalid values in ref_len: {invalid_ref}\n"
|
|
189
|
+
f"Invalid values in hyp_len: {invalid_hyp}"
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
# accept len and str and test if it makes sense
|
|
193
|
+
def _validate_conversion(
|
|
194
|
+
self,
|
|
195
|
+
str_rep: str,
|
|
196
|
+
len_rep: list[int],
|
|
197
|
+
):
|
|
198
|
+
assert len(str_rep) == sum(len_rep) - 1
|
|
199
|
+
|
|
200
|
+
def _nltk_default_k(self, ref_str, boundary: str = "1") -> int:
|
|
201
|
+
n = len(ref_str)
|
|
202
|
+
b = ref_str.count(boundary)
|
|
203
|
+
|
|
204
|
+
if b == 0:
|
|
205
|
+
raise ValueError("Reference contains no boundaries")
|
|
206
|
+
|
|
207
|
+
return int(round(n / (2.0 * b)))
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass(slots=True, frozen=True)
|
|
6
|
+
class SegmentationResult:
|
|
7
|
+
"""Result returned by segmenters.
|
|
8
|
+
|
|
9
|
+
Attributes:
|
|
10
|
+
segment_lengths:
|
|
11
|
+
Length of each predicted segment.
|
|
12
|
+
|
|
13
|
+
runtime:
|
|
14
|
+
Segmentation runtime in seconds.
|
|
15
|
+
|
|
16
|
+
metadata:
|
|
17
|
+
Optional algorithm-specific intermediate values.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
segment_lengths: list[int]
|
|
21
|
+
runtime: float
|
|
22
|
+
# method: str
|
|
23
|
+
metadata: dict = field(default_factory=dict)
|
|
24
|
+
|
|
25
|
+
def iter_spans(self):
|
|
26
|
+
start = 0
|
|
27
|
+
|
|
28
|
+
for length in self.segment_lengths:
|
|
29
|
+
end = start + length
|
|
30
|
+
yield start, end
|
|
31
|
+
start = end
|
|
32
|
+
|
|
33
|
+
# TODO: redesign akwared api
|
|
34
|
+
# curr: result.split(document.sentences)
|
|
35
|
+
# goal: result.split()
|
|
36
|
+
def split(self, sentences: list[str]) -> list[list[str]]:
|
|
37
|
+
chunks = []
|
|
38
|
+
|
|
39
|
+
for start, end in self.iter_spans():
|
|
40
|
+
chunks.append(sentences[start:end])
|
|
41
|
+
|
|
42
|
+
return chunks
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class Segmenter(ABC):
|
|
46
|
+
"""Abstract topic segmentation interface."""
|
|
47
|
+
|
|
48
|
+
def segment(self, sentences: list[str]) -> SegmentationResult:
|
|
49
|
+
"""Segment sentences into topical regions.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
sentences:
|
|
53
|
+
Ordered sentence sequence.
|
|
54
|
+
|
|
55
|
+
Returns:
|
|
56
|
+
Segmentation result containing:
|
|
57
|
+
- segment lengths
|
|
58
|
+
- runtime information
|
|
59
|
+
- optional metadata
|
|
60
|
+
"""
|
|
61
|
+
self._validate_input(sentences)
|
|
62
|
+
return self._segment(sentences)
|
|
63
|
+
|
|
64
|
+
@abstractmethod
|
|
65
|
+
def _segment(self, sentences: list[str]) -> SegmentationResult: ...
|
|
66
|
+
|
|
67
|
+
def _validate_input(self, sentences: list[str]):
|
|
68
|
+
|
|
69
|
+
# 1. Must be a list of str --> 1.1 List, 1.2 Str
|
|
70
|
+
# 2. Cannot be empty list
|
|
71
|
+
|
|
72
|
+
if not isinstance(sentences, list):
|
|
73
|
+
raise TypeError("sentences must be a list of str")
|
|
74
|
+
|
|
75
|
+
if any(not isinstance(s, str) for s in sentences):
|
|
76
|
+
raise TypeError("all elements in sentences must be str")
|
|
77
|
+
|
|
78
|
+
if len(sentences) == 0:
|
|
79
|
+
raise ValueError("sentences must be nonempty")
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
import networkx as nx
|
|
5
|
+
import numpy as np
|
|
6
|
+
from sentence_transformers import SentenceTransformer
|
|
7
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
8
|
+
from transformers import logging as hf_logging
|
|
9
|
+
|
|
10
|
+
from .abstract import SegmentationResult, Segmenter
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GraphSegEmbeddings(Segmenter):
|
|
14
|
+
"""
|
|
15
|
+
Improved GraphSeg-style topic segmentation using sentence embeddings.
|
|
16
|
+
|
|
17
|
+
Pipeline:
|
|
18
|
+
1. Encode sentences
|
|
19
|
+
2. Build weighted similarity graph with positional decay
|
|
20
|
+
3. Detect communities
|
|
21
|
+
4. Convert communities -> ordered labels
|
|
22
|
+
5. Smooth labels with neighborhood majority vote
|
|
23
|
+
6. Convert smoothed labels -> segment lengths
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, model: str):
|
|
27
|
+
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
|
|
28
|
+
hf_logging.set_verbosity_error()
|
|
29
|
+
self.model = SentenceTransformer(model)
|
|
30
|
+
|
|
31
|
+
# --------------------------------------------------
|
|
32
|
+
# MAIN
|
|
33
|
+
# --------------------------------------------------
|
|
34
|
+
|
|
35
|
+
def _segment(self, sentences: list[str]) -> SegmentationResult:
|
|
36
|
+
start = time.perf_counter()
|
|
37
|
+
|
|
38
|
+
embeddings = self._vectorize(sentences)
|
|
39
|
+
sim_matrix = self._similarity_matrix(embeddings)
|
|
40
|
+
|
|
41
|
+
graph = self._build_graph(sim_matrix)
|
|
42
|
+
communities = self._communities(graph)
|
|
43
|
+
|
|
44
|
+
labels = self._communities_to_labels(
|
|
45
|
+
communities=communities,
|
|
46
|
+
n_sentences=len(sentences),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
smoothed_labels = self._smooth_labels(labels, window=2)
|
|
50
|
+
|
|
51
|
+
segment_lengths = self._labels_to_segments(smoothed_labels)
|
|
52
|
+
|
|
53
|
+
runtime = time.perf_counter() - start
|
|
54
|
+
|
|
55
|
+
metadata = {
|
|
56
|
+
"embeddings": embeddings,
|
|
57
|
+
"similarity_matrix": sim_matrix,
|
|
58
|
+
"graph": graph,
|
|
59
|
+
"communities": communities,
|
|
60
|
+
"labels": labels,
|
|
61
|
+
"smoothed_labels": smoothed_labels,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return SegmentationResult(
|
|
65
|
+
segment_lengths=segment_lengths,
|
|
66
|
+
runtime=runtime,
|
|
67
|
+
metadata=metadata,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
# --------------------------------------------------
|
|
71
|
+
# VECTORIZE
|
|
72
|
+
# --------------------------------------------------
|
|
73
|
+
|
|
74
|
+
def _vectorize(self, sentences: list[str]) -> np.ndarray:
|
|
75
|
+
return self.model.encode(sentences)
|
|
76
|
+
|
|
77
|
+
# --------------------------------------------------
|
|
78
|
+
# SIMILARITY
|
|
79
|
+
# --------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def _similarity_matrix(self, embeddings: np.ndarray) -> np.ndarray:
|
|
82
|
+
return cosine_similarity(embeddings)
|
|
83
|
+
|
|
84
|
+
# --------------------------------------------------
|
|
85
|
+
# GRAPH BUILDING
|
|
86
|
+
# --------------------------------------------------
|
|
87
|
+
|
|
88
|
+
def _build_graph(
|
|
89
|
+
self,
|
|
90
|
+
sim_matrix: np.ndarray,
|
|
91
|
+
max_distance: int = 15,
|
|
92
|
+
min_similarity: float = 0.30,
|
|
93
|
+
decay: float = 0.15,
|
|
94
|
+
) -> nx.Graph:
|
|
95
|
+
"""
|
|
96
|
+
Weighted graph.
|
|
97
|
+
|
|
98
|
+
Edge weight:
|
|
99
|
+
similarity * exp(-decay * distance)
|
|
100
|
+
|
|
101
|
+
Keeps softer structure than hard thresholding.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
n = len(sim_matrix)
|
|
105
|
+
G = nx.Graph()
|
|
106
|
+
|
|
107
|
+
for i in range(n):
|
|
108
|
+
G.add_node(i)
|
|
109
|
+
|
|
110
|
+
for i in range(n):
|
|
111
|
+
for j in range(i + 1, min(n, i + max_distance + 1)):
|
|
112
|
+
sim = float(sim_matrix[i, j])
|
|
113
|
+
|
|
114
|
+
if sim < min_similarity:
|
|
115
|
+
continue
|
|
116
|
+
|
|
117
|
+
distance = abs(i - j)
|
|
118
|
+
weight = sim * np.exp(-decay * distance)
|
|
119
|
+
|
|
120
|
+
if weight > 0:
|
|
121
|
+
G.add_edge(i, j, weight=weight)
|
|
122
|
+
|
|
123
|
+
return G
|
|
124
|
+
|
|
125
|
+
# --------------------------------------------------
|
|
126
|
+
# COMMUNITIES
|
|
127
|
+
# --------------------------------------------------
|
|
128
|
+
|
|
129
|
+
def _communities(self, graph: nx.Graph) -> list[list[int]]:
|
|
130
|
+
"""
|
|
131
|
+
Greedy modularity clustering.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
communities = nx.algorithms.community.greedy_modularity_communities(
|
|
135
|
+
graph,
|
|
136
|
+
weight="weight",
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
return [sorted(list(c)) for c in communities]
|
|
140
|
+
|
|
141
|
+
# --------------------------------------------------
|
|
142
|
+
# COMMUNITIES -> LABELS
|
|
143
|
+
# --------------------------------------------------
|
|
144
|
+
|
|
145
|
+
def _communities_to_labels(
|
|
146
|
+
self,
|
|
147
|
+
communities: list[list[int]],
|
|
148
|
+
n_sentences: int,
|
|
149
|
+
) -> list[int]:
|
|
150
|
+
"""
|
|
151
|
+
Convert unordered communities into sentence-order labels.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
labels = [-1] * n_sentences
|
|
155
|
+
|
|
156
|
+
for cid, community in enumerate(communities):
|
|
157
|
+
for idx in community:
|
|
158
|
+
labels[idx] = cid
|
|
159
|
+
|
|
160
|
+
# isolated nodes remain unique singleton labels
|
|
161
|
+
next_label = len(communities)
|
|
162
|
+
|
|
163
|
+
for i in range(n_sentences):
|
|
164
|
+
if labels[i] == -1:
|
|
165
|
+
labels[i] = next_label
|
|
166
|
+
next_label += 1
|
|
167
|
+
|
|
168
|
+
return labels
|
|
169
|
+
|
|
170
|
+
# --------------------------------------------------
|
|
171
|
+
# LABEL SMOOTHING
|
|
172
|
+
# --------------------------------------------------
|
|
173
|
+
|
|
174
|
+
def _smooth_labels(
|
|
175
|
+
self,
|
|
176
|
+
labels: list[int],
|
|
177
|
+
window: int = 2,
|
|
178
|
+
) -> list[int]:
|
|
179
|
+
"""
|
|
180
|
+
Majority-vote smoothing.
|
|
181
|
+
|
|
182
|
+
Example:
|
|
183
|
+
A A B A A -> A A A A A
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
smoothed = labels.copy()
|
|
187
|
+
n = len(labels)
|
|
188
|
+
|
|
189
|
+
for i in range(n):
|
|
190
|
+
left = max(0, i - window)
|
|
191
|
+
right = min(n, i + window + 1)
|
|
192
|
+
|
|
193
|
+
neighborhood = labels[left:right]
|
|
194
|
+
|
|
195
|
+
values, counts = np.unique(neighborhood, return_counts=True)
|
|
196
|
+
majority = values[np.argmax(counts)]
|
|
197
|
+
|
|
198
|
+
smoothed[i] = int(majority)
|
|
199
|
+
|
|
200
|
+
return smoothed
|
|
201
|
+
|
|
202
|
+
# --------------------------------------------------
|
|
203
|
+
# LABELS -> SEGMENTS
|
|
204
|
+
# --------------------------------------------------
|
|
205
|
+
|
|
206
|
+
def _labels_to_segments(self, labels: list[int]) -> list[int]:
|
|
207
|
+
"""
|
|
208
|
+
Convert contiguous labels into segment lengths.
|
|
209
|
+
"""
|
|
210
|
+
|
|
211
|
+
lengths = []
|
|
212
|
+
|
|
213
|
+
current = labels[0]
|
|
214
|
+
run = 1
|
|
215
|
+
|
|
216
|
+
for i in range(1, len(labels)):
|
|
217
|
+
if labels[i] == current:
|
|
218
|
+
run += 1
|
|
219
|
+
else:
|
|
220
|
+
lengths.append(run)
|
|
221
|
+
run = 1
|
|
222
|
+
current = labels[i]
|
|
223
|
+
|
|
224
|
+
lengths.append(run)
|
|
225
|
+
|
|
226
|
+
return lengths
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
from scipy.ndimage import uniform_filter1d
|
|
6
|
+
from scipy.signal import find_peaks
|
|
7
|
+
from sentence_transformers import SentenceTransformer
|
|
8
|
+
from sklearn.metrics.pairwise import cosine_similarity
|
|
9
|
+
from transformers import logging as hf_logging
|
|
10
|
+
|
|
11
|
+
from .abstract import SegmentationResult, Segmenter
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TextTilingEmbeddings(Segmenter):
|
|
15
|
+
"""Embedding-based TextTiling segmentation.
|
|
16
|
+
|
|
17
|
+
This segmenter replaces lexical similarity with sentence
|
|
18
|
+
embedding similarity computed from a transformer model.
|
|
19
|
+
|
|
20
|
+
Pipeline:
|
|
21
|
+
1. Encode sentences into embeddings
|
|
22
|
+
2. Compute adjacent cosine similarities
|
|
23
|
+
3. Smooth similarity signal
|
|
24
|
+
4. Detect valley boundaries
|
|
25
|
+
5. Convert boundaries into segment lengths
|
|
26
|
+
|
|
27
|
+
Args:
|
|
28
|
+
model:
|
|
29
|
+
SentenceTransformer model name.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(self, model: str):
|
|
33
|
+
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
|
|
34
|
+
hf_logging.set_verbosity_error()
|
|
35
|
+
self.model = SentenceTransformer(model)
|
|
36
|
+
|
|
37
|
+
def _segment(self, sentences: list[str]) -> SegmentationResult:
|
|
38
|
+
start = time.perf_counter()
|
|
39
|
+
|
|
40
|
+
embeddings = self._vectorize(self.model, sentences)
|
|
41
|
+
similarities = self._similarity(embeddings)
|
|
42
|
+
smoothed = self._smooth(similarities)
|
|
43
|
+
boundaries = self._boundaries(signal=smoothed)
|
|
44
|
+
segment_lengths = self._postprocess(
|
|
45
|
+
boundaries=boundaries, n_sentences=len(sentences)
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
end = time.perf_counter()
|
|
49
|
+
runtime = end - start
|
|
50
|
+
|
|
51
|
+
metadata = {
|
|
52
|
+
"embeddings": embeddings,
|
|
53
|
+
"similarities": similarities,
|
|
54
|
+
"smoothed": smoothed,
|
|
55
|
+
"boundaries": boundaries,
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return SegmentationResult(
|
|
59
|
+
segment_lengths=segment_lengths, runtime=runtime, metadata=metadata
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
def _vectorize(self, model, sentences):
|
|
63
|
+
embeddings = model.encode(sentences)
|
|
64
|
+
return embeddings
|
|
65
|
+
|
|
66
|
+
def _similarity(self, embeddings):
|
|
67
|
+
sims = []
|
|
68
|
+
|
|
69
|
+
for i in range(len(embeddings) - 1):
|
|
70
|
+
sim = cosine_similarity([embeddings[i]], [embeddings[i + 1]])[0][0]
|
|
71
|
+
|
|
72
|
+
sims.append(sim)
|
|
73
|
+
|
|
74
|
+
return sims
|
|
75
|
+
|
|
76
|
+
def _smooth(self, similarities):
|
|
77
|
+
# TODO: make smoothing windows a class state
|
|
78
|
+
smoothed = uniform_filter1d(similarities, size=2)
|
|
79
|
+
return smoothed
|
|
80
|
+
|
|
81
|
+
# TODO: fix thresholding
|
|
82
|
+
# TODO: adaptive thresholding
|
|
83
|
+
|
|
84
|
+
def _boundaries(self, signal, alpha=0.5, plimit=0.1):
|
|
85
|
+
"""
|
|
86
|
+
signal: smoothed similarity curve
|
|
87
|
+
alpha : threshold parameter
|
|
88
|
+
plimit: minimum candidate score
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
signal = np.asarray(signal)
|
|
92
|
+
|
|
93
|
+
valleys, _ = find_peaks(-signal)
|
|
94
|
+
|
|
95
|
+
candidates = []
|
|
96
|
+
|
|
97
|
+
for v in valleys:
|
|
98
|
+
left_peak = np.max(signal[: v + 1])
|
|
99
|
+
right_peak = np.max(signal[v:])
|
|
100
|
+
|
|
101
|
+
score = 0.5 * (left_peak + right_peak - 2 * signal[v])
|
|
102
|
+
|
|
103
|
+
if score >= plimit:
|
|
104
|
+
candidates.append((v, score))
|
|
105
|
+
|
|
106
|
+
if not candidates:
|
|
107
|
+
return []
|
|
108
|
+
|
|
109
|
+
vals = np.array([s for _, s in candidates])
|
|
110
|
+
|
|
111
|
+
mu = np.mean(vals)
|
|
112
|
+
sigma = np.std(vals)
|
|
113
|
+
|
|
114
|
+
threshold = mu - alpha * sigma
|
|
115
|
+
|
|
116
|
+
boundaries = [idx for idx, score in candidates if score >= threshold]
|
|
117
|
+
|
|
118
|
+
return boundaries
|
|
119
|
+
|
|
120
|
+
def _postprocess(self, boundaries, n_sentences):
|
|
121
|
+
boundaries = sorted(int(b) for b in boundaries)
|
|
122
|
+
|
|
123
|
+
lengths = []
|
|
124
|
+
start = 0
|
|
125
|
+
|
|
126
|
+
for b in boundaries:
|
|
127
|
+
end = b + 1 # boundary after sentence b
|
|
128
|
+
lengths.append(end - start)
|
|
129
|
+
start = end
|
|
130
|
+
|
|
131
|
+
lengths.append(n_sentences - start)
|
|
132
|
+
|
|
133
|
+
return lengths
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: dobermann
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Add your description here
|
|
5
|
+
Author: Paddy Denier
|
|
6
|
+
Author-email: Paddy Denier <202327146+paddydenier@users.noreply.github.com>
|
|
7
|
+
Requires-Dist: spacy>=3.8.14
|
|
8
|
+
Requires-Dist: nltk>=3.9.4
|
|
9
|
+
Requires-Dist: sentence-transformers>=5.4.1
|
|
10
|
+
Requires-Python: >=3.12
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# dobermann
|
|
14
|
+
|
|
15
|
+
Dobermann is a modern Python library for discourse segmentation, evaluation, and visualization, combining classical computational linguistics literature with contemporary NLP and embedding-based methods.
|
|
16
|
+
|
|
17
|
+

|
|
18
|
+

|
|
19
|
+
|
|
20
|
+
## Features
|
|
21
|
+
|
|
22
|
+
- ✂️ Discourse Segmentation Algorithms from Computational Linguistics Literature
|
|
23
|
+
- 📈 Built-In Datasets and Evaluation Metrics
|
|
24
|
+
- 👀 Visualization Tools
|
|
25
|
+
- 📖 Free and Open Source
|
|
26
|
+
|
|
27
|
+
## Installation
|
|
28
|
+
|
|
29
|
+
Dobermann can conveniently be installed through the pip package manager:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pip install dobermann
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Alternatively, clone the repository to access the full range of tools or contribute to the project:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
git clone https://github.com/paddydenier/dobermann.git
|
|
39
|
+
cd dobermann
|
|
40
|
+
pip install -e .
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
TODO: explain constraints.
|
|
44
|
+
|
|
45
|
+
## Usage
|
|
46
|
+
|
|
47
|
+
Minimal segmentation and evaluation workflow example:
|
|
48
|
+
|
|
49
|
+
<!-- BEGIN:quickstart -->
|
|
50
|
+
```python
|
|
51
|
+
from dobermann import Document, GraphSegEmbeddings, SegmentationEvaluator
|
|
52
|
+
|
|
53
|
+
text = "Cats are domesticated mammals that are commonly kept as pets. They belong to the family Felidae and are known for their agility. Cats have sharp claws and excellent night vision. Many cats communicate using vocalizations such as meowing and purring. Dogs are also domesticated mammals and are among the most common household pets. They belong to the family Canidae and have a strong sense of smell. Dogs have been bred for many different purposes, including hunting and herding. Many dogs are trained to assist humans in various tasks. Python is a high-level programming language used for many different applications. It is widely used in web development, data science, and automation. Python uses indentation to define blocks of code. Functions in Python can accept arguments and return values. A function is defined using the def keyword. Python also provides many built-in data structures such as lists and dictionaries."
|
|
54
|
+
document = Document.from_text(text)
|
|
55
|
+
|
|
56
|
+
segmenter = GraphSegEmbeddings("all-MiniLM-L6-v2")
|
|
57
|
+
segmentation_result = segmenter.segment(document.sentences)
|
|
58
|
+
|
|
59
|
+
print(segmentation_result.split(document.sentences))
|
|
60
|
+
```
|
|
61
|
+
<!-- END:quickstart -->
|
|
62
|
+
|
|
63
|
+
## API Usage
|
|
64
|
+
|
|
65
|
+
For integration into existing pipelines, Dobermann provides a standardized FastAPI interface for exposing its segmentation capabilities through a REST API.
|
|
66
|
+
|
|
67
|
+
### Starting the API
|
|
68
|
+
|
|
69
|
+
On default port 8000:
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
make backend
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
On custom port, e.g., 8081:
|
|
76
|
+
```bash
|
|
77
|
+
make backend PORT=8081
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## Contribution and Development Workflows
|
|
81
|
+
|
|
82
|
+
TODO: explain makefile.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
dobermann/__init__.py,sha256=5x3yE-EJJrmoVW_-1l1F5gkH-kG38E8yO1_m5KLr7CQ,399
|
|
2
|
+
dobermann/document.py,sha256=_JLFgIC0oapKTmxfDDdiIHPqdzish7KBKMjTVi1GMaA,433
|
|
3
|
+
dobermann/evaluators/__init__.py,sha256=UPWFVoOom_Kelf6CBirBSvlm4CMsaF0h2Krs36e6sEo,133
|
|
4
|
+
dobermann/evaluators/segmentation_evaluator.py,sha256=QNVewQPA-H_9gDfnUc07HWrU2FAQNDxEWGdY8A36iDU,6556
|
|
5
|
+
dobermann/segmenters/__init__.py,sha256=CW7LkKFTx-Fv3taCy6HzVYqhwTlV0Ymnee0jotJ0tFs,244
|
|
6
|
+
dobermann/segmenters/abstract.py,sha256=T_1J4g1NjkcvdUoVx-iuigLjJARFdAs-gsb86sPh1wo,2108
|
|
7
|
+
dobermann/segmenters/graphseg_embeddings.py,sha256=Jfg-iCI_SBjrNhwUPjrzhOpPta6pNklr9416MRD7uyQ,6159
|
|
8
|
+
dobermann/segmenters/texttiling_embeddings.py,sha256=snQtRK4DmOWAYUwiBM_65Vi6CXV40N0Guz8AWem2S2o,3697
|
|
9
|
+
dobermann-0.1.0.dist-info/WHEEL,sha256=4OL6Foqnnp3xRY5wMkjgc25_i5YJC6dKsC6LPcjqEoU,80
|
|
10
|
+
dobermann-0.1.0.dist-info/entry_points.txt,sha256=j7mpNERTXQapqbQODRuJq9jjYg3KkuDsIbU7ft8jqPk,46
|
|
11
|
+
dobermann-0.1.0.dist-info/METADATA,sha256=5zrkV_Cq_Jh3liyYPkK4iFp8_f35tLNNN8Gh3BVUe-U,3025
|
|
12
|
+
dobermann-0.1.0.dist-info/RECORD,,
|