tmplot 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tmplot-0.3.0 → tmplot-0.4.0}/MANIFEST.in +1 -0
- {tmplot-0.3.0/src/tmplot.egg-info → tmplot-0.4.0}/PKG-INFO +15 -36
- {tmplot-0.3.0 → tmplot-0.4.0}/README.md +2 -2
- {tmplot-0.3.0 → tmplot-0.4.0}/pyproject.toml +13 -14
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/__init__.py +1 -1
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/_distance.py +88 -18
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/_helpers.py +149 -68
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/_metrics.py +17 -5
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/_report.py +67 -36
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/_stability.py +26 -12
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot/_vis.py +14 -9
- {tmplot-0.3.0 → tmplot-0.4.0/src/tmplot.egg-info}/PKG-INFO +15 -36
- tmplot-0.4.0/src/tmplot.egg-info/requires.txt +19 -0
- tmplot-0.3.0/src/tmplot.egg-info/requires.txt +0 -17
- {tmplot-0.3.0 → tmplot-0.4.0}/LICENSE +0 -0
- {tmplot-0.3.0 → tmplot-0.4.0}/setup.cfg +0 -0
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot.egg-info/SOURCES.txt +0 -0
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot.egg-info/dependency_links.txt +0 -0
- {tmplot-0.3.0 → tmplot-0.4.0}/src/tmplot.egg-info/top_level.txt +0 -0
|
@@ -1,56 +1,35 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tmplot
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Visualization of Topic Modeling Results
|
|
5
5
|
Author-email: Maksim Terpilovskii <maximtrp@gmail.com>
|
|
6
|
-
License: MIT
|
|
7
|
-
|
|
8
|
-
Copyright (c) 2021 Maksim Terpilowski
|
|
9
|
-
|
|
10
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
-
in the Software without restriction, including without limitation the rights
|
|
13
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
-
furnished to do so, subject to the following conditions:
|
|
16
|
-
|
|
17
|
-
The above copyright notice and this permission notice shall be included in all
|
|
18
|
-
copies or substantial portions of the Software.
|
|
19
|
-
|
|
20
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
-
SOFTWARE.
|
|
27
|
-
|
|
6
|
+
License-Expression: MIT
|
|
28
7
|
Project-URL: homepage, https://github.com/maximtrp/tmplot
|
|
29
8
|
Project-URL: documentation, https://tmplot.readthedocs.io/
|
|
30
9
|
Keywords: data science,data analytics
|
|
31
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
32
10
|
Classifier: Operating System :: OS Independent
|
|
33
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
34
11
|
Classifier: Programming Language :: Python :: 3.9
|
|
35
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
36
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
37
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
38
15
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
39
16
|
Classifier: Topic :: Text Processing :: General
|
|
40
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.9
|
|
41
18
|
Description-Content-Type: text/markdown
|
|
42
19
|
License-File: LICENSE
|
|
43
|
-
Requires-Dist: numpy
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist:
|
|
46
|
-
Requires-Dist:
|
|
47
|
-
Requires-Dist:
|
|
48
|
-
Requires-Dist:
|
|
49
|
-
Requires-Dist:
|
|
50
|
-
Requires-Dist: tqdm
|
|
20
|
+
Requires-Dist: numpy>=1.23
|
|
21
|
+
Requires-Dist: pandas>=1.5
|
|
22
|
+
Requires-Dist: scikit-learn>=1.4
|
|
23
|
+
Requires-Dist: scipy>=1.9
|
|
24
|
+
Requires-Dist: altair>=5
|
|
25
|
+
Requires-Dist: ipywidgets>=8
|
|
26
|
+
Requires-Dist: IPython>=8
|
|
27
|
+
Requires-Dist: tqdm>=4.64
|
|
51
28
|
Provides-Extra: test
|
|
52
29
|
Requires-Dist: pytest; extra == "test"
|
|
53
30
|
Requires-Dist: coverage; extra == "test"
|
|
31
|
+
Requires-Dist: build; extra == "test"
|
|
32
|
+
Requires-Dist: twine; extra == "test"
|
|
54
33
|
Provides-Extra: models
|
|
55
34
|
Requires-Dist: tomotopy>=0.8.0; extra == "models"
|
|
56
35
|
Requires-Dist: gensim; extra == "models"
|
|
@@ -170,7 +149,7 @@ saliency = tmp.get_salient_terms(phi, theta)
|
|
|
170
149
|
|
|
171
150
|
```python
|
|
172
151
|
# Create topic distance matrix with different metrics
|
|
173
|
-
topic_dists = tmp.get_topics_dist(phi, method='
|
|
152
|
+
topic_dists = tmp.get_topics_dist(phi, method='jsd')
|
|
174
153
|
|
|
175
154
|
# Generate coordinates with custom algorithm
|
|
176
155
|
coords = tmp.get_topics_scatter(topic_dists, theta, method='tsne')
|
|
@@ -181,7 +160,7 @@ tmp.plot_scatter_topics(coords, topic=3) # Highlight topic 3
|
|
|
181
160
|
|
|
182
161
|
- [Complete Tutorial](https://tmplot.readthedocs.io/en/latest/tutorial.html) - Step-by-step guide
|
|
183
162
|
- [API Reference](https://tmplot.readthedocs.io/) - Full documentation
|
|
184
|
-
- [
|
|
163
|
+
- [Tutorial Notebook](docs/source/tutorial.ipynb) - Jupyter walkthrough
|
|
185
164
|
|
|
186
165
|
## Requirements
|
|
187
166
|
|
|
@@ -111,7 +111,7 @@ saliency = tmp.get_salient_terms(phi, theta)
|
|
|
111
111
|
|
|
112
112
|
```python
|
|
113
113
|
# Create topic distance matrix with different metrics
|
|
114
|
-
topic_dists = tmp.get_topics_dist(phi, method='
|
|
114
|
+
topic_dists = tmp.get_topics_dist(phi, method='jsd')
|
|
115
115
|
|
|
116
116
|
# Generate coordinates with custom algorithm
|
|
117
117
|
coords = tmp.get_topics_scatter(topic_dists, theta, method='tsne')
|
|
@@ -122,7 +122,7 @@ tmp.plot_scatter_topics(coords, topic=3) # Highlight topic 3
|
|
|
122
122
|
|
|
123
123
|
- [Complete Tutorial](https://tmplot.readthedocs.io/en/latest/tutorial.html) - Step-by-step guide
|
|
124
124
|
- [API Reference](https://tmplot.readthedocs.io/) - Full documentation
|
|
125
|
-
- [
|
|
125
|
+
- [Tutorial Notebook](docs/source/tutorial.ipynb) - Jupyter walkthrough
|
|
126
126
|
|
|
127
127
|
## Requirements
|
|
128
128
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools>=
|
|
2
|
+
requires = ["setuptools>=77.0", "wheel"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
@@ -7,8 +7,9 @@ name = "tmplot"
|
|
|
7
7
|
dynamic = ["version"]
|
|
8
8
|
description = "Visualization of Topic Modeling Results"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
requires-python = ">=3.
|
|
11
|
-
license
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
12
13
|
authors = [
|
|
13
14
|
{ name = "Maksim Terpilovskii", email = "maximtrp@gmail.com" },
|
|
14
15
|
]
|
|
@@ -17,9 +18,7 @@ keywords = [
|
|
|
17
18
|
"data analytics"
|
|
18
19
|
]
|
|
19
20
|
classifiers = [
|
|
20
|
-
"License :: OSI Approved :: MIT License",
|
|
21
21
|
"Operating System :: OS Independent",
|
|
22
|
-
"Programming Language :: Python :: 3.8",
|
|
23
22
|
"Programming Language :: Python :: 3.9",
|
|
24
23
|
"Programming Language :: Python :: 3.10",
|
|
25
24
|
"Programming Language :: Python :: 3.11",
|
|
@@ -31,18 +30,18 @@ urls.homepage = "https://github.com/maximtrp/tmplot"
|
|
|
31
30
|
urls.documentation = "https://tmplot.readthedocs.io/"
|
|
32
31
|
|
|
33
32
|
dependencies = [
|
|
34
|
-
"numpy",
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
-
"
|
|
41
|
-
"tqdm",
|
|
33
|
+
"numpy>=1.23",
|
|
34
|
+
"pandas>=1.5",
|
|
35
|
+
"scikit-learn>=1.4",
|
|
36
|
+
"scipy>=1.9",
|
|
37
|
+
"altair>=5",
|
|
38
|
+
"ipywidgets>=8",
|
|
39
|
+
"IPython>=8",
|
|
40
|
+
"tqdm>=4.64",
|
|
42
41
|
]
|
|
43
42
|
|
|
44
43
|
[project.optional-dependencies]
|
|
45
|
-
test = ["pytest", "coverage"]
|
|
44
|
+
test = ["pytest", "coverage", "build", "twine"]
|
|
46
45
|
models = ["tomotopy>=0.8.0", "gensim", "bitermplus"]
|
|
47
46
|
|
|
48
47
|
[tool.setuptools.dynamic]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
__all__ = ["get_topics_dist", "get_topics_scatter", "get_top_topic_words"]
|
|
2
2
|
from typing import Optional, Union, List
|
|
3
|
+
from inspect import signature
|
|
3
4
|
from itertools import combinations
|
|
4
5
|
from pandas import DataFrame, Index
|
|
5
6
|
import numpy as np
|
|
@@ -15,14 +16,22 @@ from sklearn.manifold import (
|
|
|
15
16
|
from ._helpers import calc_topics_marg_probs
|
|
16
17
|
|
|
17
18
|
|
|
19
|
+
EPSILON = 1e-64
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _positive_probabilities(values: np.ndarray) -> np.ndarray:
|
|
23
|
+
values = np.clip(np.asarray(values, dtype=float), EPSILON, None)
|
|
24
|
+
return values / values.sum()
|
|
25
|
+
|
|
26
|
+
|
|
18
27
|
def _dist_klb(a1: np.ndarray, a2: np.ndarray):
|
|
19
|
-
|
|
20
|
-
return dist[np.isfinite(dist)].sum()
|
|
28
|
+
return kl_div(_positive_probabilities(a1), _positive_probabilities(a2)).sum()
|
|
21
29
|
|
|
22
30
|
|
|
23
31
|
def _dist_sklb(a1: np.ndarray, a2: np.ndarray):
|
|
24
|
-
|
|
25
|
-
|
|
32
|
+
a1_safe = _positive_probabilities(a1)
|
|
33
|
+
a2_safe = _positive_probabilities(a2)
|
|
34
|
+
return (kl_div(a1_safe, a2_safe) + kl_div(a2_safe, a1_safe)).sum()
|
|
26
35
|
|
|
27
36
|
|
|
28
37
|
def _dist_jsd(a1: np.ndarray, a2: np.ndarray):
|
|
@@ -32,13 +41,9 @@ def _dist_jsd(a1: np.ndarray, a2: np.ndarray):
|
|
|
32
41
|
|
|
33
42
|
|
|
34
43
|
def _dist_jef(a1: np.ndarray, a2: np.ndarray):
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
return
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
# Small value for numerical stability
|
|
41
|
-
EPSILON = 1e-64
|
|
44
|
+
a1_safe = _positive_probabilities(a1)
|
|
45
|
+
a2_safe = _positive_probabilities(a2)
|
|
46
|
+
return ((a1_safe - a2_safe) * (np.log(a1_safe) - np.log(a2_safe))).sum()
|
|
42
47
|
|
|
43
48
|
|
|
44
49
|
def _dist_hel(a1: np.ndarray, a2: np.ndarray):
|
|
@@ -124,8 +129,12 @@ def get_topics_dist(
|
|
|
124
129
|
"jac": _dist_jac,
|
|
125
130
|
}
|
|
126
131
|
|
|
132
|
+
if method not in dist_funcs:
|
|
133
|
+
raise ValueError(
|
|
134
|
+
f"Unknown distance method {method!r}; choose from {sorted(dist_funcs)}"
|
|
135
|
+
)
|
|
136
|
+
_dist_func = dist_funcs[method]
|
|
127
137
|
for i, j in topics_pairs:
|
|
128
|
-
_dist_func = dist_funcs.get(method, "sklb")
|
|
129
138
|
topics_dists[((i, j), (j, i))] = _dist_func(
|
|
130
139
|
phi_copy[:, i], phi_copy[:, j], **kwargs
|
|
131
140
|
)
|
|
@@ -133,6 +142,17 @@ def get_topics_dist(
|
|
|
133
142
|
return topics_dists
|
|
134
143
|
|
|
135
144
|
|
|
145
|
+
def _classical_mds(distances: np.ndarray) -> np.ndarray:
|
|
146
|
+
count = distances.shape[0]
|
|
147
|
+
centering = np.eye(count) - np.ones((count, count)) / count
|
|
148
|
+
gram = -0.5 * centering @ (distances**2) @ centering
|
|
149
|
+
eigenvalues, eigenvectors = np.linalg.eigh(gram)
|
|
150
|
+
positive = eigenvalues > np.finfo(float).eps
|
|
151
|
+
if not positive.any():
|
|
152
|
+
return np.zeros((count, 1))
|
|
153
|
+
return eigenvectors[:, positive] * np.sqrt(eigenvalues[positive])
|
|
154
|
+
|
|
155
|
+
|
|
136
156
|
def get_topics_scatter(
|
|
137
157
|
topic_dists: np.ndarray,
|
|
138
158
|
theta: np.ndarray,
|
|
@@ -164,11 +184,39 @@ def get_topics_scatter(
|
|
|
164
184
|
DataFrame
|
|
165
185
|
Topics scatter coordinates.
|
|
166
186
|
"""
|
|
167
|
-
|
|
168
|
-
|
|
187
|
+
topic_dists = np.asarray(topic_dists, dtype=float)
|
|
188
|
+
theta = np.asarray(theta, dtype=float)
|
|
189
|
+
if topic_dists.ndim != 2 or topic_dists.shape[0] != topic_dists.shape[1]:
|
|
190
|
+
raise ValueError("topic_dists must be a square 2D matrix")
|
|
191
|
+
if not np.isfinite(topic_dists).all():
|
|
192
|
+
raise ValueError("topic_dists must contain only finite values")
|
|
193
|
+
if not np.allclose(topic_dists, topic_dists.T):
|
|
194
|
+
raise ValueError("topic_dists must be symmetric")
|
|
195
|
+
if theta.ndim != 2 or theta.shape[0] != topic_dists.shape[0]:
|
|
196
|
+
raise ValueError("theta topics dimension must match topic_dists")
|
|
197
|
+
if topic_dists.shape[0] < 2:
|
|
198
|
+
raise ValueError("at least two topics are required for scatter coordinates")
|
|
199
|
+
|
|
200
|
+
valid_methods = ["tsne", "sem", "mds", "lle", "ltsa", "isomap"]
|
|
201
|
+
if method not in valid_methods:
|
|
202
|
+
raise ValueError(
|
|
203
|
+
f"Unknown scatter method {method!r}; choose from {valid_methods}"
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
if topic_dists.shape[0] == 2:
|
|
207
|
+
half_distance = topic_dists[0, 1] / 2
|
|
208
|
+
coords = np.array([[-half_distance, 0.0], [half_distance, 0.0]])
|
|
209
|
+
topics_xy = DataFrame(coords, columns=Index(["x", "y"]))
|
|
210
|
+
topics_xy["topic"] = topics_xy.index.astype(int)
|
|
211
|
+
topics_xy["size"] = calc_topics_marg_probs(theta) * 100
|
|
212
|
+
return topics_xy
|
|
213
|
+
|
|
214
|
+
method_kws = dict(method_kws or {})
|
|
215
|
+
method_kws.setdefault("n_components", 2)
|
|
169
216
|
|
|
170
217
|
if method == "tsne":
|
|
171
|
-
method_kws.setdefault("
|
|
218
|
+
method_kws.setdefault("metric", "precomputed")
|
|
219
|
+
method_kws.setdefault("init", "random")
|
|
172
220
|
method_kws.setdefault("learning_rate", "auto")
|
|
173
221
|
method_kws.setdefault("perplexity", min(50, max(topic_dists.shape[0] // 2, 1)))
|
|
174
222
|
transformer = TSNE(**method_kws)
|
|
@@ -176,25 +224,46 @@ def get_topics_scatter(
|
|
|
176
224
|
elif method == "sem":
|
|
177
225
|
method_kws.setdefault("affinity", "precomputed")
|
|
178
226
|
transformer = SpectralEmbedding(**method_kws)
|
|
227
|
+
nonzero = topic_dists[topic_dists > 0]
|
|
228
|
+
scale = np.median(nonzero) if nonzero.size else 1.0
|
|
229
|
+
transform_input = np.exp(-((topic_dists / scale) ** 2))
|
|
230
|
+
np.fill_diagonal(transform_input, 1.0)
|
|
179
231
|
|
|
180
232
|
elif method == "mds":
|
|
181
|
-
|
|
233
|
+
mds_params = signature(MDS.__init__).parameters
|
|
234
|
+
if "metric_mds" in mds_params:
|
|
235
|
+
# scikit-learn >= 1.9 deprecated `dissimilarity` in favor of `metric`
|
|
236
|
+
method_kws.setdefault("metric", "precomputed")
|
|
237
|
+
else:
|
|
238
|
+
method_kws.setdefault("dissimilarity", "precomputed")
|
|
182
239
|
method_kws.setdefault("normalized_stress", "auto")
|
|
183
240
|
method_kws.setdefault("n_init", 1)
|
|
241
|
+
if "init" in mds_params:
|
|
242
|
+
# `init` is a constructor argument only since scikit-learn 1.9
|
|
243
|
+
method_kws.setdefault("init", "random")
|
|
184
244
|
transformer = MDS(**method_kws)
|
|
185
245
|
|
|
186
246
|
elif method == "lle":
|
|
187
247
|
method_kws["method"] = "standard"
|
|
248
|
+
method_kws.setdefault("n_neighbors", min(5, topic_dists.shape[0] - 1))
|
|
188
249
|
transformer = LocallyLinearEmbedding(**method_kws)
|
|
250
|
+
transform_input = _classical_mds(topic_dists)
|
|
189
251
|
|
|
190
252
|
elif method == "ltsa":
|
|
191
253
|
method_kws["method"] = "ltsa"
|
|
254
|
+
method_kws.setdefault("n_neighbors", min(5, topic_dists.shape[0] - 1))
|
|
192
255
|
transformer = LocallyLinearEmbedding(**method_kws)
|
|
256
|
+
transform_input = _classical_mds(topic_dists)
|
|
193
257
|
|
|
194
258
|
elif method == "isomap":
|
|
259
|
+
method_kws.setdefault("metric", "precomputed")
|
|
260
|
+
method_kws.setdefault("n_neighbors", min(5, topic_dists.shape[0] - 1))
|
|
195
261
|
transformer = Isomap(**method_kws)
|
|
196
262
|
|
|
197
|
-
|
|
263
|
+
else:
|
|
264
|
+
raise AssertionError("validated scatter method was not handled")
|
|
265
|
+
|
|
266
|
+
coords = transformer.fit_transform(locals().get("transform_input", topic_dists))
|
|
198
267
|
|
|
199
268
|
topics_xy = DataFrame(coords, columns=Index(["x", "y"]))
|
|
200
269
|
topics_xy["topic"] = topics_xy.index.astype(int)
|
|
@@ -229,6 +298,7 @@ def get_top_topic_words(
|
|
|
229
298
|
DataFrame
|
|
230
299
|
Words with highest probabilities in all (or selected) topics.
|
|
231
300
|
"""
|
|
232
|
-
|
|
301
|
+
selected_topics = phi.columns if topics_idx is None else topics_idx
|
|
302
|
+
return phi.loc[:, selected_topics].apply(
|
|
233
303
|
lambda x: x.sort_values(ascending=False).head(words_num).index, axis=0
|
|
234
304
|
)
|
|
@@ -9,43 +9,65 @@ __all__ = [
|
|
|
9
9
|
"calc_topics_marg_probs",
|
|
10
10
|
"calc_terms_probs_ratio",
|
|
11
11
|
]
|
|
12
|
-
from warnings import warn
|
|
13
12
|
from importlib.util import find_spec
|
|
13
|
+
from warnings import warn
|
|
14
14
|
from typing import Union, Optional, Sequence, List
|
|
15
15
|
from functools import partial
|
|
16
|
-
from
|
|
17
|
-
from numpy import ndarray, zeros, argsort, array, arange, vstack
|
|
16
|
+
from numpy import ndarray, zeros, array, arange, vstack
|
|
18
17
|
from numpy import log as nplog
|
|
18
|
+
import numpy as np
|
|
19
19
|
from pandas import concat, Series, DataFrame
|
|
20
20
|
|
|
21
21
|
tomotopy_installed = find_spec("tomotopy")
|
|
22
22
|
if tomotopy_installed:
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
23
|
+
try:
|
|
24
|
+
from tomotopy import (
|
|
25
|
+
LDAModel as tomotopyLDA,
|
|
26
|
+
LLDAModel as tomotopyLLDA,
|
|
27
|
+
CTModel as tomotopyCT,
|
|
28
|
+
DMRModel as tomotopyDMR,
|
|
29
|
+
HDPModel as tomotopyHDP,
|
|
30
|
+
PTModel as tomotopyPT,
|
|
31
|
+
SLDAModel as tomotopySLDA,
|
|
32
|
+
GDMRModel as tomotopyGDMR,
|
|
33
|
+
)
|
|
34
|
+
except (ImportError, OSError):
|
|
35
|
+
tomotopy_installed = None
|
|
33
36
|
|
|
34
37
|
gensim_installed = find_spec("gensim")
|
|
35
38
|
if gensim_installed:
|
|
36
|
-
|
|
37
|
-
|
|
39
|
+
try:
|
|
40
|
+
from gensim.models.ldamodel import LdaModel as gensimLDA
|
|
41
|
+
from gensim.models.ldamulticore import LdaMulticore as gensimLDAMC
|
|
42
|
+
except (ImportError, OSError):
|
|
43
|
+
gensim_installed = None
|
|
38
44
|
|
|
39
45
|
bitermplus_installed = find_spec("bitermplus")
|
|
40
46
|
if bitermplus_installed:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
47
|
+
try:
|
|
48
|
+
from bitermplus._btm import BTM
|
|
49
|
+
except (ImportError, OSError):
|
|
50
|
+
bitermplus_installed = None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _warn_missing_model_packages() -> None:
|
|
54
|
+
missing = [
|
|
55
|
+
name
|
|
56
|
+
for name, installed in (
|
|
57
|
+
("tomotopy", tomotopy_installed),
|
|
58
|
+
("gensim", gensim_installed),
|
|
59
|
+
("bitermplus", bitermplus_installed),
|
|
60
|
+
)
|
|
61
|
+
if not installed
|
|
62
|
+
]
|
|
63
|
+
if missing:
|
|
64
|
+
packages = ", ".join(missing)
|
|
65
|
+
warn(
|
|
66
|
+
f"Optional model adapter packages are not installed: {packages}. "
|
|
67
|
+
f"Install the required adapter to analyze its models.",
|
|
68
|
+
UserWarning,
|
|
69
|
+
stacklevel=2,
|
|
70
|
+
)
|
|
49
71
|
|
|
50
72
|
|
|
51
73
|
def get_phi(model: object, vocabulary: Optional[Sequence] = None) -> DataFrame:
|
|
@@ -81,7 +103,9 @@ def get_phi(model: object, vocabulary: Optional[Sequence] = None) -> DataFrame:
|
|
|
81
103
|
|
|
82
104
|
elif _is_gensim(model):
|
|
83
105
|
phi = DataFrame(model.get_topics().T)
|
|
84
|
-
if vocabulary:
|
|
106
|
+
if vocabulary is not None:
|
|
107
|
+
if len(vocabulary) != phi.shape[0]:
|
|
108
|
+
raise ValueError("vocabulary length must match the number of words")
|
|
85
109
|
phi.index = vocabulary
|
|
86
110
|
|
|
87
111
|
elif _is_btmplus(model):
|
|
@@ -91,6 +115,9 @@ def get_phi(model: object, vocabulary: Optional[Sequence] = None) -> DataFrame:
|
|
|
91
115
|
phi.index.name = "words"
|
|
92
116
|
phi.columns.name = "topics"
|
|
93
117
|
|
|
118
|
+
if phi is None:
|
|
119
|
+
_warn_missing_model_packages()
|
|
120
|
+
raise ValueError(f"Unsupported model type: {type(model)}")
|
|
94
121
|
return phi
|
|
95
122
|
|
|
96
123
|
|
|
@@ -108,7 +135,6 @@ def _is_tomotopy(model: object) -> bool:
|
|
|
108
135
|
]
|
|
109
136
|
return any(map(partial(isinstance, model), tomotopy_models))
|
|
110
137
|
|
|
111
|
-
__warn_package_installation("tomotopy")
|
|
112
138
|
return False
|
|
113
139
|
|
|
114
140
|
|
|
@@ -117,7 +143,6 @@ def _is_gensim(model: object) -> bool:
|
|
|
117
143
|
gensim_models = [gensimLDA, gensimLDAMC]
|
|
118
144
|
return any(map(partial(isinstance, model), gensim_models))
|
|
119
145
|
|
|
120
|
-
__warn_package_installation("gensim")
|
|
121
146
|
return False
|
|
122
147
|
|
|
123
148
|
|
|
@@ -125,7 +150,6 @@ def _is_btmplus(model: object) -> bool:
|
|
|
125
150
|
if bitermplus_installed:
|
|
126
151
|
return isinstance(model, BTM)
|
|
127
152
|
|
|
128
|
-
__warn_package_installation("bitermplus")
|
|
129
153
|
return False
|
|
130
154
|
|
|
131
155
|
|
|
@@ -159,15 +183,16 @@ def get_theta(model: object, corpus: Optional[List] = None) -> Optional[DataFram
|
|
|
159
183
|
if len(corpus) == 0:
|
|
160
184
|
raise ValueError("corpus cannot be empty")
|
|
161
185
|
tdd = list(map(model.get_document_topics, corpus))
|
|
162
|
-
|
|
186
|
+
theta_values = zeros((len(tdd), model.num_topics))
|
|
163
187
|
for doc_id, doc_topic in enumerate(tdd):
|
|
164
188
|
for topic_id, topic_prob in doc_topic:
|
|
165
|
-
|
|
166
|
-
theta =
|
|
189
|
+
theta_values[doc_id, topic_id] = topic_prob
|
|
190
|
+
theta = DataFrame(theta_values.T)
|
|
167
191
|
|
|
168
192
|
elif _is_btmplus(model):
|
|
169
193
|
theta = DataFrame(model.matrix_topics_docs_)
|
|
170
194
|
else:
|
|
195
|
+
_warn_missing_model_packages()
|
|
171
196
|
raise ValueError(f"Unsupported model type: {type(model)}")
|
|
172
197
|
|
|
173
198
|
if isinstance(theta, DataFrame):
|
|
@@ -236,12 +261,22 @@ def get_top_docs(
|
|
|
236
261
|
if all([model is None, theta is None]):
|
|
237
262
|
raise ValueError("Please pass a model or a theta matrix to function")
|
|
238
263
|
|
|
239
|
-
if
|
|
264
|
+
if theta is None:
|
|
240
265
|
theta = get_theta(model, corpus=corpus).values
|
|
241
266
|
|
|
267
|
+
theta = array(theta)
|
|
268
|
+
if theta.ndim != 2:
|
|
269
|
+
raise ValueError("theta must be a 2D topics x documents matrix")
|
|
270
|
+
if len(docs) != theta.shape[1]:
|
|
271
|
+
raise ValueError("docs length must match theta documents dimension")
|
|
272
|
+
if docs_num <= 0:
|
|
273
|
+
raise ValueError("docs_num must be positive")
|
|
274
|
+
|
|
242
275
|
def _select_docs(docs, theta, topic_id: int):
|
|
243
276
|
probs = theta[topic_id, :]
|
|
244
|
-
|
|
277
|
+
count = min(docs_num, probs.size)
|
|
278
|
+
idx = np.argpartition(probs, -count)[-count:]
|
|
279
|
+
idx = idx[np.argsort(probs[idx])[::-1]]
|
|
245
280
|
result = Series(list(map(lambda x: docs[x], idx)))
|
|
246
281
|
result.name = f"topic{topic_id}"
|
|
247
282
|
return result
|
|
@@ -271,10 +306,14 @@ def calc_topics_marg_probs(
|
|
|
271
306
|
theta_arr = array(theta)
|
|
272
307
|
if theta_arr.size == 0:
|
|
273
308
|
raise ValueError("theta matrix cannot be empty")
|
|
309
|
+
if theta_arr.ndim != 2:
|
|
310
|
+
raise ValueError("theta matrix must be a 2D array")
|
|
311
|
+
if not np.isfinite(theta_arr).all() or np.any(theta_arr < 0):
|
|
312
|
+
raise ValueError("theta matrix must contain finite non-negative values")
|
|
274
313
|
|
|
275
314
|
p_t = theta_arr.sum(axis=1)
|
|
276
315
|
total_sum = p_t.sum()
|
|
277
|
-
if total_sum
|
|
316
|
+
if total_sum <= 0:
|
|
278
317
|
raise ValueError("theta matrix contains all zeros - cannot normalize")
|
|
279
318
|
|
|
280
319
|
p_t /= total_sum
|
|
@@ -311,12 +350,25 @@ def calc_terms_marg_probs(
|
|
|
311
350
|
|
|
312
351
|
if phi_arr.size == 0:
|
|
313
352
|
raise ValueError("phi matrix cannot be empty")
|
|
353
|
+
if phi_arr.ndim != 2:
|
|
354
|
+
raise ValueError("phi matrix must be a 2D array")
|
|
314
355
|
if p_t_arr.size == 0:
|
|
315
356
|
raise ValueError("p_t array cannot be empty")
|
|
357
|
+
if p_t_arr.ndim != 1:
|
|
358
|
+
raise ValueError("p_t array must be a 1D array")
|
|
359
|
+
if not np.isfinite(phi_arr).all() or np.any(phi_arr < 0):
|
|
360
|
+
raise ValueError("phi matrix must contain finite non-negative values")
|
|
361
|
+
if not np.isfinite(p_t_arr).all() or np.any(p_t_arr < 0):
|
|
362
|
+
raise ValueError("p_t must contain finite non-negative values")
|
|
316
363
|
if phi_arr.shape[1] != p_t_arr.shape[0]:
|
|
317
|
-
raise ValueError(
|
|
364
|
+
raise ValueError(
|
|
365
|
+
f"phi topics dimension {phi_arr.shape[1]} must match p_t length {p_t_arr.shape[0]}"
|
|
366
|
+
)
|
|
318
367
|
|
|
319
|
-
|
|
368
|
+
p_t_sum = p_t_arr.sum()
|
|
369
|
+
if p_t_sum <= 0:
|
|
370
|
+
raise ValueError("p_t must have positive total probability")
|
|
371
|
+
p_w = (phi_arr * (p_t_arr / p_t_sum)).sum(axis=1)
|
|
320
372
|
if word_id is not None:
|
|
321
373
|
if word_id < 0 or word_id >= len(p_w):
|
|
322
374
|
raise IndexError(f"word_id {word_id} out of bounds for {len(p_w)} words")
|
|
@@ -343,33 +395,34 @@ def get_salient_terms(phi: ndarray, theta: ndarray) -> ndarray:
|
|
|
343
395
|
numpy.ndarray
|
|
344
396
|
Terms saliency values.
|
|
345
397
|
"""
|
|
398
|
+
phi = array(phi, dtype=float)
|
|
399
|
+
theta = array(theta, dtype=float)
|
|
346
400
|
if phi.size == 0 or theta.size == 0:
|
|
347
401
|
raise ValueError("phi and theta matrices cannot be empty")
|
|
348
402
|
if phi.shape[1] != theta.shape[0]:
|
|
349
|
-
raise ValueError(
|
|
403
|
+
raise ValueError(
|
|
404
|
+
f"phi topics dimension {phi.shape[1]} must match theta topics dimension {theta.shape[0]}"
|
|
405
|
+
)
|
|
350
406
|
|
|
351
407
|
p_t = calc_topics_marg_probs(theta)
|
|
352
408
|
p_w = calc_terms_marg_probs(phi, p_t)
|
|
353
409
|
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
if _p_tw(phi, w, t) > 0 and p_t[t] > 0
|
|
366
|
-
else 0 # Handle log(0) cases
|
|
367
|
-
for t in range(phi.shape[1])
|
|
368
|
-
)
|
|
369
|
-
)
|
|
370
|
-
for w in range(phi.shape[0])
|
|
371
|
-
]
|
|
410
|
+
p_tw = np.divide(
|
|
411
|
+
phi * p_t,
|
|
412
|
+
p_w[:, None],
|
|
413
|
+
out=np.zeros_like(phi, dtype=float),
|
|
414
|
+
where=p_w[:, None] > 0,
|
|
415
|
+
)
|
|
416
|
+
ratio = np.divide(
|
|
417
|
+
p_tw,
|
|
418
|
+
p_t,
|
|
419
|
+
out=np.ones_like(p_tw),
|
|
420
|
+
where=p_t > 0,
|
|
372
421
|
)
|
|
422
|
+
contributions = np.zeros_like(p_tw)
|
|
423
|
+
positive = p_tw > 0
|
|
424
|
+
contributions[positive] = p_tw[positive] * np.log(ratio[positive])
|
|
425
|
+
saliency = p_w * contributions.sum(axis=1)
|
|
373
426
|
# saliency(term w) = frequency(w)
|
|
374
427
|
# * [sum_t p(t | w) * log(p(t | w)/p(t))] for topics t
|
|
375
428
|
# p(t | w) = p(w | t) * p(t) / p(w)
|
|
@@ -377,7 +430,11 @@ def get_salient_terms(phi: ndarray, theta: ndarray) -> ndarray:
|
|
|
377
430
|
|
|
378
431
|
|
|
379
432
|
def calc_terms_probs_ratio(
|
|
380
|
-
phi: DataFrame,
|
|
433
|
+
phi: DataFrame,
|
|
434
|
+
topic: int,
|
|
435
|
+
terms_num: int = 30,
|
|
436
|
+
lambda_: float = 0.6,
|
|
437
|
+
p_t: Optional[ndarray] = None,
|
|
381
438
|
) -> DataFrame:
|
|
382
439
|
"""Get terms conditional and marginal probabilities.
|
|
383
440
|
|
|
@@ -406,6 +463,8 @@ def calc_terms_probs_ratio(
|
|
|
406
463
|
pandas.DataFrame
|
|
407
464
|
Words conditional and marginal probabilities.
|
|
408
465
|
"""
|
|
466
|
+
if not 0 <= lambda_ <= 1:
|
|
467
|
+
raise ValueError("lambda_ must be between 0 and 1")
|
|
409
468
|
p_cond_name = "Conditional term probability, p(w | t)"
|
|
410
469
|
p_cond = (
|
|
411
470
|
phi.iloc[:, topic].rename(p_cond_name)
|
|
@@ -414,29 +473,31 @@ def calc_terms_probs_ratio(
|
|
|
414
473
|
)
|
|
415
474
|
|
|
416
475
|
p_marg_name = "Marginal term probability, p(w)"
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
)
|
|
476
|
+
topic_probs = np.full(phi.shape[1], 1 / phi.shape[1]) if p_t is None else array(p_t)
|
|
477
|
+
marginal = calc_terms_marg_probs(phi, topic_probs)
|
|
478
|
+
index = phi.index if isinstance(phi, DataFrame) else None
|
|
479
|
+
p_marg = Series(marginal, index=index, name=p_marg_name)
|
|
422
480
|
|
|
423
481
|
terms_probs = concat((p_marg, p_cond), axis=1)
|
|
424
|
-
relevant_idx = get_relevant_terms(phi, topic, lambda_).index
|
|
482
|
+
relevant_idx = get_relevant_terms(phi, topic, lambda_, p_t=p_t).index
|
|
425
483
|
terms_probs_slice = terms_probs.loc[relevant_idx].head(terms_num)
|
|
426
484
|
|
|
427
485
|
return (
|
|
428
|
-
terms_probs_slice.
|
|
486
|
+
terms_probs_slice.rename_axis("Terms")
|
|
487
|
+
.reset_index(drop=False)
|
|
429
488
|
.melt(
|
|
430
|
-
id_vars=[
|
|
489
|
+
id_vars=["Terms"],
|
|
431
490
|
var_name="Type",
|
|
432
491
|
value_name="Probability",
|
|
433
492
|
)
|
|
434
|
-
.rename(columns={terms_probs_slice.index.name: "Terms"})
|
|
435
493
|
)
|
|
436
494
|
|
|
437
495
|
|
|
438
496
|
def get_relevant_terms(
|
|
439
|
-
phi: Union[ndarray, DataFrame],
|
|
497
|
+
phi: Union[ndarray, DataFrame],
|
|
498
|
+
topic: int,
|
|
499
|
+
lambda_: float = 0.6,
|
|
500
|
+
p_t: Optional[ndarray] = None,
|
|
440
501
|
) -> Series:
|
|
441
502
|
"""Select relevant terms.
|
|
442
503
|
|
|
@@ -463,10 +524,30 @@ def get_relevant_terms(
|
|
|
463
524
|
pandas.Series
|
|
464
525
|
Terms sorted by relevance (descendingly).
|
|
465
526
|
"""
|
|
466
|
-
|
|
527
|
+
if not 0 <= lambda_ <= 1:
|
|
528
|
+
raise ValueError("lambda_ must be between 0 and 1")
|
|
529
|
+
phi_arr = array(phi, dtype=float)
|
|
530
|
+
if phi_arr.ndim != 2:
|
|
531
|
+
raise ValueError("phi must be a 2D words x topics matrix")
|
|
532
|
+
if not 0 <= topic < phi_arr.shape[1]:
|
|
533
|
+
raise IndexError("topic is out of bounds")
|
|
534
|
+
topic_probs = (
|
|
535
|
+
np.full(phi_arr.shape[1], 1 / phi_arr.shape[1])
|
|
536
|
+
if p_t is None
|
|
537
|
+
else array(p_t, dtype=float)
|
|
538
|
+
)
|
|
539
|
+
if topic_probs.ndim != 1 or topic_probs.shape[0] != phi_arr.shape[1]:
|
|
540
|
+
raise ValueError("p_t length must match the number of topics")
|
|
541
|
+
p_marg = calc_terms_marg_probs(phi_arr, topic_probs)
|
|
542
|
+
phi_topic = phi_arr[:, topic]
|
|
467
543
|
|
|
468
544
|
# relevance = lambda * log(p(w | t)) + (1 - lambda) * log(p(w | t) / p(w))
|
|
469
|
-
|
|
470
|
-
phi_topic
|
|
545
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
546
|
+
relevance = lambda_ * nplog(phi_topic) + (1 - lambda_) * nplog(
|
|
547
|
+
np.divide(phi_topic, p_marg, out=np.zeros_like(phi_topic), where=p_marg > 0)
|
|
548
|
+
)
|
|
549
|
+
relevance = Series(
|
|
550
|
+
relevance,
|
|
551
|
+
index=phi.index if isinstance(phi, DataFrame) else None,
|
|
471
552
|
)
|
|
472
553
|
return relevance.sort_values(ascending=False)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"""Metrics module"""
|
|
2
|
+
|
|
2
3
|
from math import log
|
|
3
4
|
import numpy as np
|
|
4
|
-
|
|
5
|
+
|
|
6
|
+
__all__ = ["entropy"]
|
|
5
7
|
|
|
6
8
|
|
|
7
9
|
def entropy(phi: np.ndarray, max_probs: bool = False):
|
|
@@ -40,6 +42,12 @@ def entropy(phi: np.ndarray, max_probs: bool = False):
|
|
|
40
42
|
>>> # Entropy calculation
|
|
41
43
|
>>> entropy = tmp.entropy(phi)
|
|
42
44
|
"""
|
|
45
|
+
phi = np.asarray(phi, dtype=float)
|
|
46
|
+
if phi.ndim != 2 or 0 in phi.shape:
|
|
47
|
+
raise ValueError("phi must be a non-empty 2D topics x words matrix")
|
|
48
|
+
if not np.isfinite(phi).all() or np.any(phi < 0):
|
|
49
|
+
raise ValueError("phi must contain finite non-negative probabilities")
|
|
50
|
+
|
|
43
51
|
# Terms number
|
|
44
52
|
words_num = phi.shape[1]
|
|
45
53
|
# Topics number
|
|
@@ -53,14 +61,18 @@ def entropy(phi: np.ndarray, max_probs: bool = False):
|
|
|
53
61
|
p_max = np.max(phi, axis=0)
|
|
54
62
|
|
|
55
63
|
# Select the probabilities larger than thresh
|
|
56
|
-
p_max_mask = p_max
|
|
64
|
+
p_max_mask = p_max >= thresh
|
|
57
65
|
word_ratio = p_max_mask.sum()
|
|
58
66
|
sum_prob = p_max[p_max_mask].sum()
|
|
59
67
|
|
|
60
68
|
else:
|
|
61
69
|
# Select the probabilities larger than thresh
|
|
62
|
-
|
|
63
|
-
|
|
70
|
+
mask = phi >= thresh
|
|
71
|
+
sum_prob = phi[mask].sum()
|
|
72
|
+
word_ratio = np.count_nonzero(mask)
|
|
73
|
+
|
|
74
|
+
if word_ratio == 0 or sum_prob <= 0:
|
|
75
|
+
raise ValueError("phi does not contain probabilities at or above the threshold")
|
|
64
76
|
|
|
65
77
|
# Shannon entropy
|
|
66
78
|
shannon = log(word_ratio / (words_num * topics_num))
|
|
@@ -75,6 +87,6 @@ def entropy(phi: np.ndarray, max_probs: bool = False):
|
|
|
75
87
|
if topics_num == 1:
|
|
76
88
|
renyi = free_energy / topics_num
|
|
77
89
|
else:
|
|
78
|
-
renyi = free_energy / (topics_num-1)
|
|
90
|
+
renyi = free_energy / (topics_num - 1)
|
|
79
91
|
|
|
80
92
|
return renyi
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
__all__ = ["prepare_coords", "report"]
|
|
2
|
-
import warnings
|
|
3
2
|
from typing import Dict, Optional, Sequence, List
|
|
4
3
|
from copy import deepcopy
|
|
5
4
|
from IPython.display import display
|
|
@@ -7,10 +6,13 @@ from ipywidgets import widgets as wdg
|
|
|
7
6
|
from pandas import DataFrame
|
|
8
7
|
from ._distance import get_topics_dist, get_topics_scatter
|
|
9
8
|
from ._vis import plot_scatter_topics, plot_terms, plot_docs
|
|
10
|
-
from ._helpers import
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
from ._helpers import (
|
|
10
|
+
calc_terms_probs_ratio,
|
|
11
|
+
calc_topics_marg_probs,
|
|
12
|
+
get_phi,
|
|
13
|
+
get_theta,
|
|
14
|
+
get_top_docs,
|
|
15
|
+
)
|
|
14
16
|
|
|
15
17
|
|
|
16
18
|
def prepare_coords(
|
|
@@ -44,7 +46,10 @@ def prepare_coords(
|
|
|
44
46
|
theta = get_theta(model, corpus=corpus)
|
|
45
47
|
topics_dists = get_topics_dist(phi, **dist_kws)
|
|
46
48
|
topics_coords = get_topics_scatter(topics_dists, theta, **scatter_kws)
|
|
47
|
-
|
|
49
|
+
selected_labels = theta.index if labels is None else labels
|
|
50
|
+
if len(selected_labels) != len(topics_coords):
|
|
51
|
+
raise ValueError("labels length must match the number of topics")
|
|
52
|
+
topics_coords["label"] = selected_labels
|
|
48
53
|
return topics_coords
|
|
49
54
|
|
|
50
55
|
|
|
@@ -114,7 +119,9 @@ def report(
|
|
|
114
119
|
if not topics_kws
|
|
115
120
|
else deepcopy(topics_kws)
|
|
116
121
|
)
|
|
117
|
-
_coords_kws = {"corpus": corpus}
|
|
122
|
+
_coords_kws = {"corpus": corpus}
|
|
123
|
+
if coords_kws:
|
|
124
|
+
_coords_kws.update(deepcopy(coords_kws))
|
|
118
125
|
_words_kws = (
|
|
119
126
|
{"chart_kws": {"height": height, "width": width}}
|
|
120
127
|
if not words_kws
|
|
@@ -154,7 +161,7 @@ def report(
|
|
|
154
161
|
# Children widgets list init
|
|
155
162
|
children = []
|
|
156
163
|
|
|
157
|
-
if "topics_coords" not in _topics_kws:
|
|
164
|
+
if show_topics and "topics_coords" not in _topics_kws:
|
|
158
165
|
topics_coords = prepare_coords(model, **_coords_kws)
|
|
159
166
|
_topics_kws.update(
|
|
160
167
|
{
|
|
@@ -165,51 +172,60 @@ def report(
|
|
|
165
172
|
}
|
|
166
173
|
)
|
|
167
174
|
|
|
168
|
-
# Cache phi matrix to avoid repeated calls
|
|
169
175
|
phi = get_phi(model)
|
|
176
|
+
theta = None
|
|
177
|
+
p_t = None
|
|
170
178
|
|
|
171
|
-
if
|
|
172
|
-
|
|
173
|
-
|
|
179
|
+
if show_words:
|
|
180
|
+
theta = get_theta(model, corpus=corpus)
|
|
181
|
+
p_t = calc_topics_marg_probs(theta)
|
|
182
|
+
if "terms_probs" not in _words_kws:
|
|
183
|
+
terms_probs = calc_terms_probs_ratio(phi, topic=0, p_t=p_t)
|
|
184
|
+
_words_kws.update({"terms_probs": terms_probs})
|
|
174
185
|
|
|
175
|
-
if
|
|
176
|
-
theta = get_theta(model, corpus=corpus)
|
|
177
|
-
_top_docs_kws.
|
|
178
|
-
|
|
179
|
-
)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
186
|
+
if show_docs:
|
|
187
|
+
theta = get_theta(model, corpus=corpus) if theta is None else theta
|
|
188
|
+
_top_docs_kws.setdefault("docs", docs)
|
|
189
|
+
_top_docs_kws.setdefault("theta", theta.values)
|
|
190
|
+
_top_docs_kws.setdefault("topics", [0])
|
|
191
|
+
_top_docs_kws.setdefault("docs_num", 2)
|
|
192
|
+
if "docs" not in _docs_kws:
|
|
193
|
+
top_docs = get_top_docs(**_top_docs_kws)
|
|
194
|
+
top_docs.columns = [""]
|
|
195
|
+
_docs_kws.update({"docs": top_docs})
|
|
183
196
|
|
|
184
197
|
# Topic selection
|
|
185
198
|
def _on_select_topic(sel):
|
|
186
|
-
topic = sel[
|
|
199
|
+
topic = sel["new"]
|
|
187
200
|
|
|
188
201
|
if show_words:
|
|
189
202
|
words_plot_output.clear_output(wait=False)
|
|
190
203
|
with words_plot_output:
|
|
191
204
|
terms_probs = calc_terms_probs_ratio(
|
|
192
|
-
phi, topic=topic, lambda_=lambda_slider.value
|
|
193
|
-
|
|
205
|
+
phi, topic=topic, lambda_=lambda_slider.value, p_t=p_t
|
|
206
|
+
)
|
|
207
|
+
_words_kws.update({"terms_probs": terms_probs})
|
|
194
208
|
display(plot_terms(**_words_kws))
|
|
195
209
|
|
|
196
210
|
if show_topics:
|
|
197
211
|
topics_plot_output.clear_output(wait=False)
|
|
198
212
|
with topics_plot_output:
|
|
199
|
-
_topics_kws.update({
|
|
213
|
+
_topics_kws.update({"topic": topic})
|
|
200
214
|
display(plot_scatter_topics(**_topics_kws))
|
|
201
215
|
|
|
202
216
|
if show_docs:
|
|
203
217
|
docs_plot_output.clear_output(wait=False)
|
|
204
218
|
with docs_plot_output:
|
|
205
|
-
_top_docs_kws.update({
|
|
219
|
+
_top_docs_kws.update({"topics": [sel["new"]]})
|
|
206
220
|
top_docs = get_top_docs(**_top_docs_kws)
|
|
207
|
-
top_docs.columns = [
|
|
208
|
-
_docs_kws.update({
|
|
221
|
+
top_docs.columns = [""]
|
|
222
|
+
_docs_kws.update({"docs": top_docs})
|
|
209
223
|
display(plot_docs(**_docs_kws))
|
|
210
224
|
|
|
211
|
-
topics_ids = list(range(
|
|
212
|
-
topics_labels
|
|
225
|
+
topics_ids = list(range(phi.shape[1]))
|
|
226
|
+
if topics_labels is not None and len(topics_labels) != len(topics_ids):
|
|
227
|
+
raise ValueError("topics_labels length must match the number of topics")
|
|
228
|
+
topics_labels = topics_ids if topics_labels is None else topics_labels
|
|
213
229
|
select_topic = wdg.Dropdown(options=list(zip(topics_labels, topics_ids)), value=0)
|
|
214
230
|
select_topic.observe(_on_select_topic, names="value")
|
|
215
231
|
select_topic_header = wdg.HTML("<b>Select a topic</b>:")
|
|
@@ -219,11 +235,24 @@ def report(
|
|
|
219
235
|
)
|
|
220
236
|
|
|
221
237
|
# Topics scatter
|
|
238
|
+
scatter_cache = {}
|
|
239
|
+
|
|
222
240
|
def _on_select_topics_method(names):
|
|
223
241
|
topics_plot_output.clear_output(wait=False)
|
|
224
242
|
with topics_plot_output:
|
|
225
|
-
|
|
226
|
-
|
|
243
|
+
if not scatter_cache:
|
|
244
|
+
scatter_theta = get_theta(model, corpus=_coords_kws.get("corpus"))
|
|
245
|
+
scatter_cache["theta"] = scatter_theta
|
|
246
|
+
scatter_cache["distances"] = get_topics_dist(
|
|
247
|
+
phi, **_coords_kws.get("dist_kws", {})
|
|
248
|
+
)
|
|
249
|
+
scatter_kws = dict(_coords_kws.get("scatter_kws", {}))
|
|
250
|
+
scatter_kws["method"] = names["new"]
|
|
251
|
+
topics_coords = get_topics_scatter(
|
|
252
|
+
scatter_cache["distances"], scatter_cache["theta"], **scatter_kws
|
|
253
|
+
)
|
|
254
|
+
labels = _coords_kws.get("labels", scatter_cache["theta"].index)
|
|
255
|
+
topics_coords["label"] = labels
|
|
227
256
|
_topics_kws.update(
|
|
228
257
|
{"topics_coords": topics_coords, "topic": select_topic.value}
|
|
229
258
|
)
|
|
@@ -243,7 +272,7 @@ def report(
|
|
|
243
272
|
topics_method = wdg.Dropdown(
|
|
244
273
|
options=options_methods,
|
|
245
274
|
value="tsne",
|
|
246
|
-
layout=wdg.Layout(width=f"{width/1.25}px"),
|
|
275
|
+
layout=wdg.Layout(width=f"{width / 1.25}px"),
|
|
247
276
|
)
|
|
248
277
|
topics_method_widget = wdg.HBox([topics_method_header, topics_method])
|
|
249
278
|
topics_method.observe(_on_select_topics_method, names="value")
|
|
@@ -262,7 +291,9 @@ def report(
|
|
|
262
291
|
lambda_ = lambda_slider.value
|
|
263
292
|
words_plot_output.clear_output(wait=False)
|
|
264
293
|
with words_plot_output:
|
|
265
|
-
terms_probs = calc_terms_probs_ratio(
|
|
294
|
+
terms_probs = calc_terms_probs_ratio(
|
|
295
|
+
phi, topic=topic, lambda_=lambda_, p_t=p_t
|
|
296
|
+
)
|
|
266
297
|
_words_kws.update({"terms_probs": terms_probs})
|
|
267
298
|
display(plot_terms(**_words_kws))
|
|
268
299
|
|
|
@@ -276,7 +307,7 @@ def report(
|
|
|
276
307
|
orientation="horizontal",
|
|
277
308
|
readout=True,
|
|
278
309
|
readout_format=".2f",
|
|
279
|
-
layout=wdg.Layout(width=f"{width/1.25}px"),
|
|
310
|
+
layout=wdg.Layout(width=f"{width / 1.25}px"),
|
|
280
311
|
)
|
|
281
312
|
lambda_slider.observe(_on_select_lambda, names="value")
|
|
282
313
|
lambda_slider_header = wdg.HTML("Lambda value:")
|
|
@@ -307,14 +338,14 @@ def report(
|
|
|
307
338
|
display(plot_docs(**_docs_kws))
|
|
308
339
|
|
|
309
340
|
docs_num_slider = wdg.IntSlider(
|
|
310
|
-
value=
|
|
341
|
+
value=_top_docs_kws["docs_num"],
|
|
311
342
|
min=1,
|
|
312
343
|
max=100,
|
|
313
344
|
continuous_update=False,
|
|
314
345
|
orientation="horizontal",
|
|
315
346
|
readout=True,
|
|
316
347
|
readout_format="d",
|
|
317
|
-
layout=wdg.Layout(width=f"{width/1.25}px"),
|
|
348
|
+
layout=wdg.Layout(width=f"{width / 1.25}px"),
|
|
318
349
|
)
|
|
319
350
|
docs_num_slider.observe(_on_select_docs_num, names="value")
|
|
320
351
|
docs_num_slider_header = wdg.HTML("Documents number:")
|
|
@@ -91,8 +91,11 @@ def get_closest_topics(
|
|
|
91
91
|
closest_topics = np.zeros(shape=(topics_num, models_num), dtype=int)
|
|
92
92
|
closest_topics[:, ref] = np.arange(topics_num)
|
|
93
93
|
|
|
94
|
-
|
|
95
|
-
|
|
94
|
+
if method not in dist_funcs:
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"Unknown distance method {method!r}; choose from {sorted(dist_funcs)}"
|
|
97
|
+
)
|
|
98
|
+
dist_func = dist_funcs[method]
|
|
96
99
|
|
|
97
100
|
# Distance values
|
|
98
101
|
dist_vals = np.zeros(shape=(topics_num, models_num), dtype=float)
|
|
@@ -109,23 +112,33 @@ def get_closest_topics(
|
|
|
109
112
|
# Get phi matrix for current model
|
|
110
113
|
current_phi = get_phi(model)
|
|
111
114
|
|
|
115
|
+
shared_words = model_ref_phi.index.intersection(current_phi.index)
|
|
116
|
+
if shared_words.empty:
|
|
117
|
+
raise ValueError("models do not have any vocabulary terms in common")
|
|
118
|
+
ref_phi = model_ref_phi.loc[shared_words]
|
|
119
|
+
current_phi = current_phi.loc[shared_words]
|
|
120
|
+
ref_sums = ref_phi.sum(axis=0)
|
|
121
|
+
current_sums = current_phi.sum(axis=0)
|
|
122
|
+
if (ref_sums <= 0).any() or (current_sums <= 0).any():
|
|
123
|
+
raise ValueError("shared vocabulary has zero probability mass for a topic")
|
|
124
|
+
ref_phi = ref_phi / ref_sums
|
|
125
|
+
current_phi = current_phi / current_sums
|
|
126
|
+
current_topics_num = current_phi.shape[1]
|
|
127
|
+
|
|
112
128
|
# Distance matrix for all topic pairs
|
|
113
|
-
all_vs_all_dists = np.zeros((topics_num,
|
|
129
|
+
all_vs_all_dists = np.zeros((topics_num, current_topics_num))
|
|
114
130
|
|
|
115
131
|
# Iterating over all topic pairs
|
|
116
132
|
for t_ref in range(topics_num):
|
|
117
|
-
for t in range(
|
|
133
|
+
for t in range(current_topics_num):
|
|
134
|
+
kwargs = {"top_words": top_words} if method == "jac" else {}
|
|
118
135
|
all_vs_all_dists[t_ref, t] = dist_func(
|
|
119
|
-
|
|
136
|
+
ref_phi.iloc[:, t_ref], current_phi.iloc[:, t], **kwargs
|
|
120
137
|
)
|
|
121
138
|
|
|
122
139
|
# Creating two arrays for the closest topics ids and distance values
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
dist_vals[:, mid] = np.max(all_vs_all_dists, axis=1)
|
|
126
|
-
else:
|
|
127
|
-
closest_topics[:, mid] = np.argmin(all_vs_all_dists, axis=1)
|
|
128
|
-
dist_vals[:, mid] = np.min(all_vs_all_dists, axis=1)
|
|
140
|
+
closest_topics[:, mid] = np.argmin(all_vs_all_dists, axis=1)
|
|
141
|
+
dist_vals[:, mid] = np.min(all_vs_all_dists, axis=1)
|
|
129
142
|
|
|
130
143
|
return closest_topics, dist_vals
|
|
131
144
|
|
|
@@ -190,7 +203,8 @@ def get_stable_topics(
|
|
|
190
203
|
... closest_topics, kldiv)
|
|
191
204
|
"""
|
|
192
205
|
dist_arr = np.asarray(dist)
|
|
193
|
-
|
|
206
|
+
max_dist = dist_arr.max()
|
|
207
|
+
dist_ready = dist_arr / max_dist if norm and max_dist > 0 else dist_arr.copy()
|
|
194
208
|
dist_ready = inverse_factor - dist_ready if inverse else dist_ready
|
|
195
209
|
mask = np.sum(np.delete(dist_ready, ref, axis=1) >= thres, axis=1) >= thres_models
|
|
196
210
|
return closest_topics[mask], dist_ready[mask]
|
|
@@ -109,15 +109,6 @@ def plot_scatter_topics(
|
|
|
109
109
|
"scale": Scale(range=[0, 3000]),
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
if not circle_enc_kws:
|
|
113
|
-
circle_enc_kws = {
|
|
114
|
-
"x": X(**x_kws),
|
|
115
|
-
"y": Y(**y_kws),
|
|
116
|
-
"size": Size(size_col, **size_kws)
|
|
117
|
-
if size_col and not topics_coords[size_col].isna().any()
|
|
118
|
-
else value(500),
|
|
119
|
-
}
|
|
120
|
-
|
|
121
112
|
if not text_kws:
|
|
122
113
|
text_kws = {"align": "center", "baseline": "middle"}
|
|
123
114
|
|
|
@@ -138,6 +129,20 @@ def plot_scatter_topics(
|
|
|
138
129
|
raise ValueError("topics_coords DataFrame cannot be empty")
|
|
139
130
|
data = topics_coords.copy()
|
|
140
131
|
|
|
132
|
+
if size_col and size_col not in data:
|
|
133
|
+
raise ValueError(f"size column {size_col!r} is missing")
|
|
134
|
+
if label_col and label_col not in data:
|
|
135
|
+
raise ValueError(f"label column {label_col!r} is missing")
|
|
136
|
+
|
|
137
|
+
if not circle_enc_kws:
|
|
138
|
+
circle_enc_kws = {
|
|
139
|
+
"x": X(**x_kws),
|
|
140
|
+
"y": Y(**y_kws),
|
|
141
|
+
"size": Size(size_col, **size_kws)
|
|
142
|
+
if size_col and not data[size_col].isna().any()
|
|
143
|
+
else value(500),
|
|
144
|
+
}
|
|
145
|
+
|
|
141
146
|
if not topic_col:
|
|
142
147
|
topic_col = "topic"
|
|
143
148
|
data = data.assign(**{topic_col: range(len(topics_coords))})
|
|
@@ -1,56 +1,35 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tmplot
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Visualization of Topic Modeling Results
|
|
5
5
|
Author-email: Maksim Terpilovskii <maximtrp@gmail.com>
|
|
6
|
-
License: MIT
|
|
7
|
-
|
|
8
|
-
Copyright (c) 2021 Maksim Terpilowski
|
|
9
|
-
|
|
10
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
-
in the Software without restriction, including without limitation the rights
|
|
13
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
-
furnished to do so, subject to the following conditions:
|
|
16
|
-
|
|
17
|
-
The above copyright notice and this permission notice shall be included in all
|
|
18
|
-
copies or substantial portions of the Software.
|
|
19
|
-
|
|
20
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
-
SOFTWARE.
|
|
27
|
-
|
|
6
|
+
License-Expression: MIT
|
|
28
7
|
Project-URL: homepage, https://github.com/maximtrp/tmplot
|
|
29
8
|
Project-URL: documentation, https://tmplot.readthedocs.io/
|
|
30
9
|
Keywords: data science,data analytics
|
|
31
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
32
10
|
Classifier: Operating System :: OS Independent
|
|
33
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
34
11
|
Classifier: Programming Language :: Python :: 3.9
|
|
35
12
|
Classifier: Programming Language :: Python :: 3.10
|
|
36
13
|
Classifier: Programming Language :: Python :: 3.11
|
|
37
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
38
15
|
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
39
16
|
Classifier: Topic :: Text Processing :: General
|
|
40
|
-
Requires-Python: >=3.
|
|
17
|
+
Requires-Python: >=3.9
|
|
41
18
|
Description-Content-Type: text/markdown
|
|
42
19
|
License-File: LICENSE
|
|
43
|
-
Requires-Dist: numpy
|
|
44
|
-
Requires-Dist:
|
|
45
|
-
Requires-Dist:
|
|
46
|
-
Requires-Dist:
|
|
47
|
-
Requires-Dist:
|
|
48
|
-
Requires-Dist:
|
|
49
|
-
Requires-Dist:
|
|
50
|
-
Requires-Dist: tqdm
|
|
20
|
+
Requires-Dist: numpy>=1.23
|
|
21
|
+
Requires-Dist: pandas>=1.5
|
|
22
|
+
Requires-Dist: scikit-learn>=1.4
|
|
23
|
+
Requires-Dist: scipy>=1.9
|
|
24
|
+
Requires-Dist: altair>=5
|
|
25
|
+
Requires-Dist: ipywidgets>=8
|
|
26
|
+
Requires-Dist: IPython>=8
|
|
27
|
+
Requires-Dist: tqdm>=4.64
|
|
51
28
|
Provides-Extra: test
|
|
52
29
|
Requires-Dist: pytest; extra == "test"
|
|
53
30
|
Requires-Dist: coverage; extra == "test"
|
|
31
|
+
Requires-Dist: build; extra == "test"
|
|
32
|
+
Requires-Dist: twine; extra == "test"
|
|
54
33
|
Provides-Extra: models
|
|
55
34
|
Requires-Dist: tomotopy>=0.8.0; extra == "models"
|
|
56
35
|
Requires-Dist: gensim; extra == "models"
|
|
@@ -170,7 +149,7 @@ saliency = tmp.get_salient_terms(phi, theta)
|
|
|
170
149
|
|
|
171
150
|
```python
|
|
172
151
|
# Create topic distance matrix with different metrics
|
|
173
|
-
topic_dists = tmp.get_topics_dist(phi, method='
|
|
152
|
+
topic_dists = tmp.get_topics_dist(phi, method='jsd')
|
|
174
153
|
|
|
175
154
|
# Generate coordinates with custom algorithm
|
|
176
155
|
coords = tmp.get_topics_scatter(topic_dists, theta, method='tsne')
|
|
@@ -181,7 +160,7 @@ tmp.plot_scatter_topics(coords, topic=3) # Highlight topic 3
|
|
|
181
160
|
|
|
182
161
|
- [Complete Tutorial](https://tmplot.readthedocs.io/en/latest/tutorial.html) - Step-by-step guide
|
|
183
162
|
- [API Reference](https://tmplot.readthedocs.io/) - Full documentation
|
|
184
|
-
- [
|
|
163
|
+
- [Tutorial Notebook](docs/source/tutorial.ipynb) - Jupyter walkthrough
|
|
185
164
|
|
|
186
165
|
## Requirements
|
|
187
166
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|