dobermann 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dobermann/__init__.py ADDED
@@ -0,0 +1,18 @@
1
+ # Public facing API
2
+ from .document import Document
3
+ from .evaluators import EvaluationResult, SegmentationEvaluator
4
+ from .segmenters import (
5
+ GraphSegEmbeddings,
6
+ SegmentationResult,
7
+ TextTilingEmbeddings,
8
+ )
9
+
10
+ __all__ = [
11
+ "Document",
12
+ "TextTiling",
13
+ "TextTilingEmbeddings",
14
+ "GraphSegEmbeddings",
15
+ "SegmentationEvaluator",
16
+ "EvaluationResult",
17
+ "SegmentationResult",
18
+ ]
dobermann/document.py ADDED
@@ -0,0 +1,20 @@
1
+ # TODO: implement document import
2
+ # create document from dataset
3
+ # create document from raw text, document from text file.
4
+ # create document from HTML
5
+ # create document from URL
6
+
7
+ from dataclasses import dataclass
8
+
9
+ import nltk
10
+
11
+
12
+ @dataclass
13
+ class Document:
14
+ sentences: list[str]
15
+
16
+ @classmethod
17
+ def from_text(cls, text: str) -> "Document":
18
+ sentences = nltk.sent_tokenize(text)
19
+
20
+ return cls(sentences=sentences)
@@ -0,0 +1,3 @@
1
+ from .segmentation_evaluator import SegmentationEvaluator, EvaluationResult
2
+
3
+ __all__ = ["SegmentationEvaluator", "EvaluationResult"]
@@ -0,0 +1,207 @@
1
+ import time
2
+ from dataclasses import dataclass
3
+
4
+ import numpy as np
5
+ from nltk.metrics import ghd as ghd
6
+ from nltk.metrics import pk as pk
7
+ from nltk.metrics import windowdiff as wd
8
+
9
+ # NOTE: only set k window size if segment lengths are highly variable
10
+
11
+
12
+ @dataclass(slots=True, frozen=True)
13
+ class EvaluationResult:
14
+ """Segmentation evluation result container.
15
+
16
+ Attributes:
17
+ hyp_str: Hypothesized segmentation string representation.
18
+ ref_str: Reference (ground truth) segmentation string representation.
19
+ pk: Pk error metrics under different settings.
20
+ wd: WindowDiff error metrics under different settings.
21
+ ghd: Generalized Hamming Distance error metric
22
+ runtime: Evaluation process execution time in seconds.
23
+ """
24
+
25
+ hyp_str: str
26
+ ref_str: str
27
+ pk: dict[str, float]
28
+ wd: dict[str, float]
29
+ ghd: float
30
+ runtime: float
31
+
32
+ def __str__(self) -> str:
33
+ """Evaluation result container summary tostring function.
34
+
35
+ Returns:
36
+ str: Multi-line human-readable and interpretable summary of evaluation result.
37
+ """
38
+ width = 34
39
+
40
+ lines = [
41
+ "EvaluationResult",
42
+ "-" * width,
43
+ f"{'Metric':<20}{'Value':>14}",
44
+ "-" * width,
45
+ f"{'Pk (small)':<20}{self.pk['small']:>14.4f}",
46
+ f"{'Pk (default)':<20}{self.pk['default']:>14.4f}",
47
+ f"{'Pk (large)':<20}{self.pk['large']:>14.4f}",
48
+ f"{'Pk (nltk)':<20}{self.pk['nltk']:>14.4f}",
49
+ "",
50
+ f"{'WD (small)':<20}{self.wd['small']:>14.4f}",
51
+ f"{'WD (default)':<20}{self.wd['default']:>14.4f}",
52
+ f"{'WD (large)':<20}{self.wd['large']:>14.4f}",
53
+ "",
54
+ f"{'GHD':<20}{self.ghd:>14.4f}",
55
+ f"{'Runtime (s)':<20}{self.runtime:>14.4f}",
56
+ "-" * width,
57
+ f"REF {self.ref_str}",
58
+ f"HYP {self.hyp_str}",
59
+ ]
60
+ return "\n".join(lines)
61
+
62
+
63
+ class SegmentationEvaluator:
64
+ """Stateful evaluator for comparing segmentation results against ground truth.
65
+
66
+ Attributes:
67
+ ins_cost: Cost of inserting a boundary.
68
+ del_cost: Cost of deleting a boundary.
69
+ shift_cost_coeff: Cost of shifting a boundary.
70
+ boundary_symbol: Character symbol that denotes a boundary.
71
+ """
72
+
73
+ def __init__(
74
+ self,
75
+ ins_cost=2.0,
76
+ del_cost=2.0,
77
+ shift_cost_coeff=1.0,
78
+ boundary_symbol="1",
79
+ ):
80
+
81
+ self.ins_cost = ins_cost
82
+ self.del_cost = del_cost
83
+ self.shift_cost_coeff = shift_cost_coeff
84
+ self.boundary_symbol = boundary_symbol
85
+
86
+ def evaluate(
87
+ self,
88
+ ref_len: list[int],
89
+ hyp_len: list[int],
90
+ ) -> EvaluationResult:
91
+ """Evaluation function
92
+
93
+ Args:
94
+ ref_len: Reference (ground truth) segment lengths.
95
+ hyp_len: Hypothesized segment lengths.
96
+
97
+ Returns:
98
+ EvaluationResult: Evaluation result container.
99
+ """
100
+ start_time = time.perf_counter()
101
+
102
+ self._validate_input(ref_len, hyp_len)
103
+
104
+ # string boundary representation
105
+ ref_str = self._lengths_to_str(ref_len)
106
+ hyp_str = self._lengths_to_str(hyp_len)
107
+
108
+ self._validate_conversion(str_rep=ref_str, len_rep=ref_len)
109
+ self._validate_conversion(str_rep=hyp_str, len_rep=hyp_len)
110
+
111
+ ghd_score = ghd(
112
+ ref_str,
113
+ hyp_str,
114
+ self.ins_cost,
115
+ self.del_cost,
116
+ self.shift_cost_coeff,
117
+ boundary=self.boundary_symbol,
118
+ )
119
+
120
+ pk_scores = {}
121
+ wd_scores = {}
122
+
123
+ k_values = self._k_values(ref_str, ref_len)
124
+ for k_name, k in k_values.items():
125
+ pk_scores[k_name] = pk(ref_str, hyp_str, k=k, boundary=self.boundary_symbol)
126
+ wd_scores[k_name] = wd(ref_str, hyp_str, k=k, boundary=self.boundary_symbol)
127
+
128
+ # optional: include nltk default (k=None)
129
+ pk_scores["nltk"] = pk(ref_str, hyp_str, k=None, boundary=self.boundary_symbol)
130
+
131
+ end_time = time.perf_counter()
132
+ runtime = end_time - start_time
133
+
134
+ return EvaluationResult(
135
+ hyp_str=hyp_str,
136
+ ref_str=ref_str,
137
+ pk=pk_scores,
138
+ wd=wd_scores,
139
+ ghd=ghd_score,
140
+ runtime=runtime,
141
+ )
142
+
143
+ def _lengths_to_str(self, lengths: list[int]) -> str:
144
+ s = []
145
+ for i, length in enumerate(lengths):
146
+ s.extend(["0"] * (length - 1))
147
+ if i != len(lengths) - 1:
148
+ s.append("1")
149
+ return "".join(s)
150
+
151
+ def _k_values(self, ref_str, ref_lengths: list[int]) -> dict[str, int]:
152
+
153
+ # percentile approach
154
+ k_small = max(1, int(round(np.percentile(ref_lengths, 25) / 2)))
155
+ # k_default = max(1, int(round(np.mean(ref_lengths) / 2)))
156
+ k_default = self._nltk_default_k(ref_str)
157
+ k_large = max(1, int(round(np.percentile(ref_lengths, 75) / 2)))
158
+
159
+ # min and max approach
160
+ # k_small = max(1, int(round(min(ref_lengths) / 2)))
161
+ # k_default = int(round((sum(ref_lengths) / len(ref_lenghts)) / 2))
162
+ # k_large = int(round(max(ref_lengths) / 2))
163
+
164
+ return {
165
+ "small": k_small,
166
+ "default": k_default,
167
+ "large": k_large,
168
+ }
169
+
170
+ def _validate_input(self, ref_len, hyp_len):
171
+ # TODO: add more meaningful error messages
172
+ ref_sum = sum(ref_len)
173
+ hyp_sum = sum(hyp_len)
174
+
175
+ if ref_sum != hyp_sum:
176
+ raise ValueError(
177
+ f"Segmentations must cover the same total length "
178
+ f"(number of sentences).\n"
179
+ f"Got: sum(ref_len)={ref_sum}, sum(hyp_len)={hyp_sum}"
180
+ )
181
+
182
+ invalid_ref = [len for len in ref_len if len <= 0]
183
+ invalid_hyp = [len for len in hyp_len if len <= 0]
184
+
185
+ if invalid_ref or invalid_hyp:
186
+ raise ValueError(
187
+ "Segment lengths must all be positive integers.\n"
188
+ f"Invalid values in ref_len: {invalid_ref}\n"
189
+ f"Invalid values in hyp_len: {invalid_hyp}"
190
+ )
191
+
192
+ # accept len and str and test if it makes sense
193
+ def _validate_conversion(
194
+ self,
195
+ str_rep: str,
196
+ len_rep: list[int],
197
+ ):
198
+ assert len(str_rep) == sum(len_rep) - 1
199
+
200
+ def _nltk_default_k(self, ref_str, boundary: str = "1") -> int:
201
+ n = len(ref_str)
202
+ b = ref_str.count(boundary)
203
+
204
+ if b == 0:
205
+ raise ValueError("Reference contains no boundaries")
206
+
207
+ return int(round(n / (2.0 * b)))
@@ -0,0 +1,9 @@
1
+ from .abstract import SegmentationResult
2
+ from .graphseg_embeddings import GraphSegEmbeddings
3
+ from .texttiling_embeddings import TextTilingEmbeddings
4
+
5
+ __all__ = [
6
+ "TextTilingEmbeddings",
7
+ "GraphSegEmbeddings",
8
+ "SegmentationResult",
9
+ ]
@@ -0,0 +1,79 @@
1
+ from abc import ABC, abstractmethod
2
+ from dataclasses import dataclass, field
3
+
4
+
5
+ @dataclass(slots=True, frozen=True)
6
+ class SegmentationResult:
7
+ """Result returned by segmenters.
8
+
9
+ Attributes:
10
+ segment_lengths:
11
+ Length of each predicted segment.
12
+
13
+ runtime:
14
+ Segmentation runtime in seconds.
15
+
16
+ metadata:
17
+ Optional algorithm-specific intermediate values.
18
+ """
19
+
20
+ segment_lengths: list[int]
21
+ runtime: float
22
+ # method: str
23
+ metadata: dict = field(default_factory=dict)
24
+
25
+ def iter_spans(self):
26
+ start = 0
27
+
28
+ for length in self.segment_lengths:
29
+ end = start + length
30
+ yield start, end
31
+ start = end
32
+
33
+ # TODO: redesign akwared api
34
+ # curr: result.split(document.sentences)
35
+ # goal: result.split()
36
+ def split(self, sentences: list[str]) -> list[list[str]]:
37
+ chunks = []
38
+
39
+ for start, end in self.iter_spans():
40
+ chunks.append(sentences[start:end])
41
+
42
+ return chunks
43
+
44
+
45
+ class Segmenter(ABC):
46
+ """Abstract topic segmentation interface."""
47
+
48
+ def segment(self, sentences: list[str]) -> SegmentationResult:
49
+ """Segment sentences into topical regions.
50
+
51
+ Args:
52
+ sentences:
53
+ Ordered sentence sequence.
54
+
55
+ Returns:
56
+ Segmentation result containing:
57
+ - segment lengths
58
+ - runtime information
59
+ - optional metadata
60
+ """
61
+ self._validate_input(sentences)
62
+ return self._segment(sentences)
63
+
64
+ @abstractmethod
65
+ def _segment(self, sentences: list[str]) -> SegmentationResult: ...
66
+
67
+ def _validate_input(self, sentences: list[str]):
68
+
69
+ # 1. Must be a list of str --> 1.1 List, 1.2 Str
70
+ # 2. Cannot be empty list
71
+
72
+ if not isinstance(sentences, list):
73
+ raise TypeError("sentences must be a list of str")
74
+
75
+ if any(not isinstance(s, str) for s in sentences):
76
+ raise TypeError("all elements in sentences must be str")
77
+
78
+ if len(sentences) == 0:
79
+ raise ValueError("sentences must be nonempty")
@@ -0,0 +1,226 @@
1
+ import logging
2
+ import time
3
+
4
+ import networkx as nx
5
+ import numpy as np
6
+ from sentence_transformers import SentenceTransformer
7
+ from sklearn.metrics.pairwise import cosine_similarity
8
+ from transformers import logging as hf_logging
9
+
10
+ from .abstract import SegmentationResult, Segmenter
11
+
12
+
13
+ class GraphSegEmbeddings(Segmenter):
14
+ """
15
+ Improved GraphSeg-style topic segmentation using sentence embeddings.
16
+
17
+ Pipeline:
18
+ 1. Encode sentences
19
+ 2. Build weighted similarity graph with positional decay
20
+ 3. Detect communities
21
+ 4. Convert communities -> ordered labels
22
+ 5. Smooth labels with neighborhood majority vote
23
+ 6. Convert smoothed labels -> segment lengths
24
+ """
25
+
26
+ def __init__(self, model: str):
27
+ logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
28
+ hf_logging.set_verbosity_error()
29
+ self.model = SentenceTransformer(model)
30
+
31
+ # --------------------------------------------------
32
+ # MAIN
33
+ # --------------------------------------------------
34
+
35
+ def _segment(self, sentences: list[str]) -> SegmentationResult:
36
+ start = time.perf_counter()
37
+
38
+ embeddings = self._vectorize(sentences)
39
+ sim_matrix = self._similarity_matrix(embeddings)
40
+
41
+ graph = self._build_graph(sim_matrix)
42
+ communities = self._communities(graph)
43
+
44
+ labels = self._communities_to_labels(
45
+ communities=communities,
46
+ n_sentences=len(sentences),
47
+ )
48
+
49
+ smoothed_labels = self._smooth_labels(labels, window=2)
50
+
51
+ segment_lengths = self._labels_to_segments(smoothed_labels)
52
+
53
+ runtime = time.perf_counter() - start
54
+
55
+ metadata = {
56
+ "embeddings": embeddings,
57
+ "similarity_matrix": sim_matrix,
58
+ "graph": graph,
59
+ "communities": communities,
60
+ "labels": labels,
61
+ "smoothed_labels": smoothed_labels,
62
+ }
63
+
64
+ return SegmentationResult(
65
+ segment_lengths=segment_lengths,
66
+ runtime=runtime,
67
+ metadata=metadata,
68
+ )
69
+
70
+ # --------------------------------------------------
71
+ # VECTORIZE
72
+ # --------------------------------------------------
73
+
74
+ def _vectorize(self, sentences: list[str]) -> np.ndarray:
75
+ return self.model.encode(sentences)
76
+
77
+ # --------------------------------------------------
78
+ # SIMILARITY
79
+ # --------------------------------------------------
80
+
81
+ def _similarity_matrix(self, embeddings: np.ndarray) -> np.ndarray:
82
+ return cosine_similarity(embeddings)
83
+
84
+ # --------------------------------------------------
85
+ # GRAPH BUILDING
86
+ # --------------------------------------------------
87
+
88
+ def _build_graph(
89
+ self,
90
+ sim_matrix: np.ndarray,
91
+ max_distance: int = 15,
92
+ min_similarity: float = 0.30,
93
+ decay: float = 0.15,
94
+ ) -> nx.Graph:
95
+ """
96
+ Weighted graph.
97
+
98
+ Edge weight:
99
+ similarity * exp(-decay * distance)
100
+
101
+ Keeps softer structure than hard thresholding.
102
+ """
103
+
104
+ n = len(sim_matrix)
105
+ G = nx.Graph()
106
+
107
+ for i in range(n):
108
+ G.add_node(i)
109
+
110
+ for i in range(n):
111
+ for j in range(i + 1, min(n, i + max_distance + 1)):
112
+ sim = float(sim_matrix[i, j])
113
+
114
+ if sim < min_similarity:
115
+ continue
116
+
117
+ distance = abs(i - j)
118
+ weight = sim * np.exp(-decay * distance)
119
+
120
+ if weight > 0:
121
+ G.add_edge(i, j, weight=weight)
122
+
123
+ return G
124
+
125
+ # --------------------------------------------------
126
+ # COMMUNITIES
127
+ # --------------------------------------------------
128
+
129
+ def _communities(self, graph: nx.Graph) -> list[list[int]]:
130
+ """
131
+ Greedy modularity clustering.
132
+ """
133
+
134
+ communities = nx.algorithms.community.greedy_modularity_communities(
135
+ graph,
136
+ weight="weight",
137
+ )
138
+
139
+ return [sorted(list(c)) for c in communities]
140
+
141
+ # --------------------------------------------------
142
+ # COMMUNITIES -> LABELS
143
+ # --------------------------------------------------
144
+
145
+ def _communities_to_labels(
146
+ self,
147
+ communities: list[list[int]],
148
+ n_sentences: int,
149
+ ) -> list[int]:
150
+ """
151
+ Convert unordered communities into sentence-order labels.
152
+ """
153
+
154
+ labels = [-1] * n_sentences
155
+
156
+ for cid, community in enumerate(communities):
157
+ for idx in community:
158
+ labels[idx] = cid
159
+
160
+ # isolated nodes remain unique singleton labels
161
+ next_label = len(communities)
162
+
163
+ for i in range(n_sentences):
164
+ if labels[i] == -1:
165
+ labels[i] = next_label
166
+ next_label += 1
167
+
168
+ return labels
169
+
170
+ # --------------------------------------------------
171
+ # LABEL SMOOTHING
172
+ # --------------------------------------------------
173
+
174
+ def _smooth_labels(
175
+ self,
176
+ labels: list[int],
177
+ window: int = 2,
178
+ ) -> list[int]:
179
+ """
180
+ Majority-vote smoothing.
181
+
182
+ Example:
183
+ A A B A A -> A A A A A
184
+ """
185
+
186
+ smoothed = labels.copy()
187
+ n = len(labels)
188
+
189
+ for i in range(n):
190
+ left = max(0, i - window)
191
+ right = min(n, i + window + 1)
192
+
193
+ neighborhood = labels[left:right]
194
+
195
+ values, counts = np.unique(neighborhood, return_counts=True)
196
+ majority = values[np.argmax(counts)]
197
+
198
+ smoothed[i] = int(majority)
199
+
200
+ return smoothed
201
+
202
+ # --------------------------------------------------
203
+ # LABELS -> SEGMENTS
204
+ # --------------------------------------------------
205
+
206
+ def _labels_to_segments(self, labels: list[int]) -> list[int]:
207
+ """
208
+ Convert contiguous labels into segment lengths.
209
+ """
210
+
211
+ lengths = []
212
+
213
+ current = labels[0]
214
+ run = 1
215
+
216
+ for i in range(1, len(labels)):
217
+ if labels[i] == current:
218
+ run += 1
219
+ else:
220
+ lengths.append(run)
221
+ run = 1
222
+ current = labels[i]
223
+
224
+ lengths.append(run)
225
+
226
+ return lengths
@@ -0,0 +1,133 @@
1
+ import logging
2
+ import time
3
+
4
+ import numpy as np
5
+ from scipy.ndimage import uniform_filter1d
6
+ from scipy.signal import find_peaks
7
+ from sentence_transformers import SentenceTransformer
8
+ from sklearn.metrics.pairwise import cosine_similarity
9
+ from transformers import logging as hf_logging
10
+
11
+ from .abstract import SegmentationResult, Segmenter
12
+
13
+
14
+ class TextTilingEmbeddings(Segmenter):
15
+ """Embedding-based TextTiling segmentation.
16
+
17
+ This segmenter replaces lexical similarity with sentence
18
+ embedding similarity computed from a transformer model.
19
+
20
+ Pipeline:
21
+ 1. Encode sentences into embeddings
22
+ 2. Compute adjacent cosine similarities
23
+ 3. Smooth similarity signal
24
+ 4. Detect valley boundaries
25
+ 5. Convert boundaries into segment lengths
26
+
27
+ Args:
28
+ model:
29
+ SentenceTransformer model name.
30
+ """
31
+
32
+ def __init__(self, model: str):
33
+ logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
34
+ hf_logging.set_verbosity_error()
35
+ self.model = SentenceTransformer(model)
36
+
37
+ def _segment(self, sentences: list[str]) -> SegmentationResult:
38
+ start = time.perf_counter()
39
+
40
+ embeddings = self._vectorize(self.model, sentences)
41
+ similarities = self._similarity(embeddings)
42
+ smoothed = self._smooth(similarities)
43
+ boundaries = self._boundaries(signal=smoothed)
44
+ segment_lengths = self._postprocess(
45
+ boundaries=boundaries, n_sentences=len(sentences)
46
+ )
47
+
48
+ end = time.perf_counter()
49
+ runtime = end - start
50
+
51
+ metadata = {
52
+ "embeddings": embeddings,
53
+ "similarities": similarities,
54
+ "smoothed": smoothed,
55
+ "boundaries": boundaries,
56
+ }
57
+
58
+ return SegmentationResult(
59
+ segment_lengths=segment_lengths, runtime=runtime, metadata=metadata
60
+ )
61
+
62
+ def _vectorize(self, model, sentences):
63
+ embeddings = model.encode(sentences)
64
+ return embeddings
65
+
66
+ def _similarity(self, embeddings):
67
+ sims = []
68
+
69
+ for i in range(len(embeddings) - 1):
70
+ sim = cosine_similarity([embeddings[i]], [embeddings[i + 1]])[0][0]
71
+
72
+ sims.append(sim)
73
+
74
+ return sims
75
+
76
+ def _smooth(self, similarities):
77
+ # TODO: make smoothing windows a class state
78
+ smoothed = uniform_filter1d(similarities, size=2)
79
+ return smoothed
80
+
81
+ # TODO: fix thresholding
82
+ # TODO: adaptive thresholding
83
+
84
+ def _boundaries(self, signal, alpha=0.5, plimit=0.1):
85
+ """
86
+ signal: smoothed similarity curve
87
+ alpha : threshold parameter
88
+ plimit: minimum candidate score
89
+ """
90
+
91
+ signal = np.asarray(signal)
92
+
93
+ valleys, _ = find_peaks(-signal)
94
+
95
+ candidates = []
96
+
97
+ for v in valleys:
98
+ left_peak = np.max(signal[: v + 1])
99
+ right_peak = np.max(signal[v:])
100
+
101
+ score = 0.5 * (left_peak + right_peak - 2 * signal[v])
102
+
103
+ if score >= plimit:
104
+ candidates.append((v, score))
105
+
106
+ if not candidates:
107
+ return []
108
+
109
+ vals = np.array([s for _, s in candidates])
110
+
111
+ mu = np.mean(vals)
112
+ sigma = np.std(vals)
113
+
114
+ threshold = mu - alpha * sigma
115
+
116
+ boundaries = [idx for idx, score in candidates if score >= threshold]
117
+
118
+ return boundaries
119
+
120
+ def _postprocess(self, boundaries, n_sentences):
121
+ boundaries = sorted(int(b) for b in boundaries)
122
+
123
+ lengths = []
124
+ start = 0
125
+
126
+ for b in boundaries:
127
+ end = b + 1 # boundary after sentence b
128
+ lengths.append(end - start)
129
+ start = end
130
+
131
+ lengths.append(n_sentences - start)
132
+
133
+ return lengths
@@ -0,0 +1,82 @@
1
+ Metadata-Version: 2.3
2
+ Name: dobermann
3
+ Version: 0.1.0
4
+ Summary: Add your description here
5
+ Author: Paddy Denier
6
+ Author-email: Paddy Denier <202327146+paddydenier@users.noreply.github.com>
7
+ Requires-Dist: spacy>=3.8.14
8
+ Requires-Dist: nltk>=3.9.4
9
+ Requires-Dist: sentence-transformers>=5.4.1
10
+ Requires-Python: >=3.12
11
+ Description-Content-Type: text/markdown
12
+
13
+ # dobermann
14
+
15
+ Dobermann is a modern Python library for discourse segmentation, evaluation, and visualization, combining classical computational linguistics literature with contemporary NLP and embedding-based methods.
16
+
17
+ ![ci workflow](https://github.com/paddydenier/dobermann/actions/workflows/ci.yml/badge.svg)
18
+ ![docs workflow](https://github.com/paddydenier/dobermann/actions/workflows/docs.yml/badge.svg)
19
+
20
+ ## Features
21
+
22
+ - ✂️ Discourse Segmentation Algorithms from Computational Linguistics Literature
23
+ - 📈 Built-In Datasets and Evaluation Metrics
24
+ - 👀 Visualization Tools
25
+ - 📖 Free and Open Source
26
+
27
+ ## Installation
28
+
29
+ Dobermann can conveniently be installed through the pip package manager:
30
+
31
+ ```bash
32
+ pip install dobermann
33
+ ```
34
+
35
+ Alternatively, clone the repository to access the full range of tools or contribute to the project:
36
+
37
+ ```bash
38
+ git clone https://github.com/paddydenier/dobermann.git
39
+ cd dobermann
40
+ pip install -e .
41
+ ```
42
+
43
+ TODO: explain constraints.
44
+
45
+ ## Usage
46
+
47
+ Minimal segmentation and evaluation workflow example:
48
+
49
+ <!-- BEGIN:quickstart -->
50
+ ```python
51
+ from dobermann import Document, GraphSegEmbeddings, SegmentationEvaluator
52
+
53
+ text = "Cats are domesticated mammals that are commonly kept as pets. They belong to the family Felidae and are known for their agility. Cats have sharp claws and excellent night vision. Many cats communicate using vocalizations such as meowing and purring. Dogs are also domesticated mammals and are among the most common household pets. They belong to the family Canidae and have a strong sense of smell. Dogs have been bred for many different purposes, including hunting and herding. Many dogs are trained to assist humans in various tasks. Python is a high-level programming language used for many different applications. It is widely used in web development, data science, and automation. Python uses indentation to define blocks of code. Functions in Python can accept arguments and return values. A function is defined using the def keyword. Python also provides many built-in data structures such as lists and dictionaries."
54
+ document = Document.from_text(text)
55
+
56
+ segmenter = GraphSegEmbeddings("all-MiniLM-L6-v2")
57
+ segmentation_result = segmenter.segment(document.sentences)
58
+
59
+ print(segmentation_result.split(document.sentences))
60
+ ```
61
+ <!-- END:quickstart -->
62
+
63
+ ## API Usage
64
+
65
+ For integration into existing pipelines, Dobermann provides a standardized FastAPI interface for exposing its segmentation capabilities through a REST API.
66
+
67
+ ### Starting the API
68
+
69
+ On default port 8000:
70
+
71
+ ```bash
72
+ make backend
73
+ ```
74
+
75
+ On custom port, e.g., 8081:
76
+ ```bash
77
+ make backend PORT=8081
78
+ ```
79
+
80
+ ## Contribution and Development Workflows
81
+
82
+ TODO: explain makefile.
@@ -0,0 +1,12 @@
1
+ dobermann/__init__.py,sha256=5x3yE-EJJrmoVW_-1l1F5gkH-kG38E8yO1_m5KLr7CQ,399
2
+ dobermann/document.py,sha256=_JLFgIC0oapKTmxfDDdiIHPqdzish7KBKMjTVi1GMaA,433
3
+ dobermann/evaluators/__init__.py,sha256=UPWFVoOom_Kelf6CBirBSvlm4CMsaF0h2Krs36e6sEo,133
4
+ dobermann/evaluators/segmentation_evaluator.py,sha256=QNVewQPA-H_9gDfnUc07HWrU2FAQNDxEWGdY8A36iDU,6556
5
+ dobermann/segmenters/__init__.py,sha256=CW7LkKFTx-Fv3taCy6HzVYqhwTlV0Ymnee0jotJ0tFs,244
6
+ dobermann/segmenters/abstract.py,sha256=T_1J4g1NjkcvdUoVx-iuigLjJARFdAs-gsb86sPh1wo,2108
7
+ dobermann/segmenters/graphseg_embeddings.py,sha256=Jfg-iCI_SBjrNhwUPjrzhOpPta6pNklr9416MRD7uyQ,6159
8
+ dobermann/segmenters/texttiling_embeddings.py,sha256=snQtRK4DmOWAYUwiBM_65Vi6CXV40N0Guz8AWem2S2o,3697
9
+ dobermann-0.1.0.dist-info/WHEEL,sha256=4OL6Foqnnp3xRY5wMkjgc25_i5YJC6dKsC6LPcjqEoU,80
10
+ dobermann-0.1.0.dist-info/entry_points.txt,sha256=j7mpNERTXQapqbQODRuJq9jjYg3KkuDsIbU7ft8jqPk,46
11
+ dobermann-0.1.0.dist-info/METADATA,sha256=5zrkV_Cq_Jh3liyYPkK4iFp8_f35tLNNN8Gh3BVUe-U,3025
12
+ dobermann-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.12.5
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ dobermann = dobermann:main
3
+