tmplot 0.1.2__tar.gz → 0.2.0__tar.gz

{tmplot-0.1.2/src/tmplot.egg-info → tmplot-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: tmplot
-Version: 0.1.2
+Version: 0.2.0
 Summary: Visualization of Topic Modeling Results
 Author-email: Maksim Terpilovskii <maximtrp@gmail.com>
 License: MIT License
@@ -96,6 +96,12 @@ Requires-Dist: bitermplus; extra == "models"
   - LocallyLinearEmbedding
   - Isomap
 
+## Donate
+
+If you find this package useful, please consider donating any amount of money. This will help me spend more time on supporting open-source software.
+
+<a href="https://www.buymeacoffee.com/maximtrp" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;" ></a>
+
 ## Installation
 
 The package can be installed from PyPi:

{tmplot-0.1.2 → tmplot-0.2.0}/README.md

@@ -38,6 +38,12 @@
   - LocallyLinearEmbedding
   - Isomap
 
+## Donate
+
+If you find this package useful, please consider donating any amount of money. This will help me spend more time on supporting open-source software.
+
+<a href="https://www.buymeacoffee.com/maximtrp" target="_blank"><img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" style="height: 60px !important;width: 217px !important;" ></a>
+
 ## Installation
 
 The package can be installed from PyPi:

{tmplot-0.1.2 → tmplot-0.2.0}/src/tmplot/__init__.py

@@ -5,4 +5,4 @@ from ._stability import *  # noqa: F401, F403
 from ._vis import *  # noqa: F401, F403
 from ._metrics import *  # noqa: F401, F403
 
-__version__ = '0.1.2'
+__version__ = '0.2.0'

{tmplot-0.1.2 → tmplot-0.2.0}/src/tmplot/_distance.py

@@ -1,13 +1,17 @@
-__all__ = [
-    'get_topics_dist', 'get_topics_scatter', 'get_top_topic_words']
-from typing import Union, List
+__all__ = ["get_topics_dist", "get_topics_scatter", "get_top_topic_words"]
+from typing import Optional, Union, List
 from itertools import combinations
-from pandas import DataFrame
+from pandas import DataFrame, Index
 import numpy as np
 from scipy.special import kl_div
 from scipy.spatial import distance
 from sklearn.manifold import (
-    TSNE, Isomap, LocallyLinearEmbedding, MDS, SpectralEmbedding)
+    TSNE,
+    Isomap,
+    LocallyLinearEmbedding,
+    MDS,
+    SpectralEmbedding,
+)
 from ._helpers import calc_topics_marg_probs
 
 
@@ -28,15 +32,14 @@ def _dist_jsd(a1: np.ndarray, a2: np.ndarray):
 
 def _dist_jef(a1: np.ndarray, a2: np.ndarray):
     vals = (a1 - a2) * (np.log(a1) - np.log(a2))
-    vals[(vals <= 0) | ~np.isfinite(vals)] = 0.
+    vals[(vals <= 0) | ~np.isfinite(vals)] = 0.0
     return vals.sum()
 
 
 def _dist_hel(a1: np.ndarray, a2: np.ndarray):
     a1[(a1 <= 0) | ~np.isfinite(a1)] = 1e-64
     a2[(a2 <= 0) | ~np.isfinite(a2)] = 1e-64
-    hel_val = distance.euclidean(
-        np.sqrt(a1), np.sqrt(a2)) / np.sqrt(2)
+    hel_val = distance.euclidean(np.sqrt(a1), np.sqrt(a2)) / np.sqrt(2)
     return hel_val
 
 
@@ -52,9 +55,9 @@ def _dist_tv(a1: np.ndarray, a2: np.ndarray):
     return dist
 
 
-def _dist_jac(a1: np.ndarray, a2: np.ndarray,  top_words=100):
-    a = np.argsort(a1)[:-top_words-1:-1]
-    b = np.argsort(a2)[:-top_words-1:-1]
+def _dist_jac(a1: np.ndarray, a2: np.ndarray, top_words=100):
+    a = np.argsort(a1)[: -top_words - 1 : -1]
+    b = np.argsort(a2)[: -top_words - 1 : -1]
     j_num = np.intersect1d(a, b, assume_unique=False).size
     j_den = np.union1d(a, b).size
     jac_val = 1 - j_num / j_den
@@ -62,9 +65,8 @@ def _dist_jac(a1: np.ndarray, a2: np.ndarray,  top_words=100):
 
 
 def get_topics_dist(
-        phi: Union[np.ndarray, DataFrame],
-        method: str = "sklb",
-        **kwargs) -> np.ndarray:
+    phi: Union[np.ndarray, DataFrame], method: str = "sklb", **kwargs
+) -> np.ndarray:
     """Finding closest topics in models.
 
     Parameters
@@ -110,16 +112,18 @@ def get_topics_dist(
     for i, j in topics_pairs:
         _dist_func = dist_funcs.get(method, "sklb")
         topics_dists[((i, j), (j, i))] = _dist_func(
-            phi_copy[:, i], phi_copy[:, j], **kwargs)
+            phi_copy[:, i], phi_copy[:, j], **kwargs
+        )
 
     return topics_dists
 
 
 def get_topics_scatter(
-        topic_dists: np.ndarray,
-        theta: np.ndarray,
-        method: str = 'tsne',
-        method_kws: dict = None) -> DataFrame:
+    topic_dists: np.ndarray,
+    theta: np.ndarray,
+    method: str = "tsne",
+    method_kws: Optional[dict] = None,
+) -> DataFrame:
     """Calculate topics coordinates for a scatter plot.
 
     Parameters
@@ -146,52 +150,52 @@ def get_topics_scatter(
         Topics scatter coordinates.
     """
     if not method_kws:
-        method_kws = {'n_components': 2}
+        method_kws = {"n_components": 2}
 
-    if method == 'tsne':
-        method_kws.setdefault('init', 'pca')
-        method_kws.setdefault('learning_rate', 'auto')
-        method_kws.setdefault(
-            'perplexity', min(50, max(topic_dists.shape[0] // 2, 1)))
+    if method == "tsne":
+        method_kws.setdefault("init", "pca")
+        method_kws.setdefault("learning_rate", "auto")
+        method_kws.setdefault("perplexity", min(50, max(topic_dists.shape[0] // 2, 1)))
         transformer = TSNE(**method_kws)
 
-    elif method == 'sem':
-        method_kws.setdefault('affinity', 'precomputed')
+    elif method == "sem":
+        method_kws.setdefault("affinity", "precomputed")
         transformer = SpectralEmbedding(**method_kws)
 
-    elif method == 'mds':
-        method_kws.setdefault('dissimilarity', 'precomputed')
-        method_kws.setdefault('normalized_stress', 'auto')
+    elif method == "mds":
+        method_kws.setdefault("dissimilarity", "precomputed")
+        method_kws.setdefault("normalized_stress", "auto")
         transformer = MDS(**method_kws)
 
-    elif method == 'lle':
-        method_kws['method'] = 'standard'
+    elif method == "lle":
+        method_kws["method"] = "standard"
         transformer = LocallyLinearEmbedding(**method_kws)
 
-    elif method == 'ltsa':
-        method_kws['method'] = 'ltsa'
+    elif method == "ltsa":
+        method_kws["method"] = "ltsa"
         transformer = LocallyLinearEmbedding(**method_kws)
 
-    elif method == 'isomap':
+    elif method == "isomap":
         transformer = Isomap(**method_kws)
 
     coords = transformer.fit_transform(topic_dists)
 
-    topics_xy = DataFrame(coords, columns=['x', 'y'])
-    topics_xy['topic'] = topics_xy.index.astype(int)
-    topics_xy['size'] = calc_topics_marg_probs(theta)
-    size_sum = topics_xy['size'].sum()
+    topics_xy = DataFrame(coords, columns=Index(["x", "y"]))
+    topics_xy["topic"] = topics_xy.index.astype(int)
+    topics_xy["size"] = calc_topics_marg_probs(theta)
+    size_sum = topics_xy["size"].sum()
     if size_sum > 0:
-        topics_xy['size'] *= (100 / topics_xy['size'].sum())
+        topics_xy["size"] *= 100 / topics_xy["size"].sum()
     else:
-        topics_xy['size'] = np.nan
+        topics_xy["size"] = np.nan
     return topics_xy
 
 
 def get_top_topic_words(
-        phi: DataFrame,
-        words_num: int = 20,
-        topics_idx: Union[List[int], np.ndarray] = None) -> DataFrame:
+    phi: DataFrame,
+    words_num: int = 20,
+    topics_idx: Optional[Union[List[int], np.ndarray]] = None,
+) -> DataFrame:
     """Select top topic words from a fitted model.
 
     Parameters
@@ -209,9 +213,6 @@ def get_top_topic_words(
     DataFrame
         Words with highest probabilities in all (or selected) topics.
     """
-    return phi.loc[:, topics_idx or phi.columns]\
-        .apply(
-            lambda x: x
-            .sort_values(ascending=False)
-            .head(words_num).index, axis=0
+    return phi.loc[:, topics_idx or phi.columns].apply(
+        lambda x: x.sort_values(ascending=False).head(words_num).index, axis=0
     )

{tmplot-0.1.2 → tmplot-0.2.0}/src/tmplot/_helpers.py

@@ -1,9 +1,14 @@
 __all__ = [
-    'get_phi', 'get_theta',
-    'get_relevant_terms', 'get_salient_terms',
-    'get_docs', 'get_top_docs',
-    'calc_terms_marg_probs', 'calc_topics_marg_probs',
-    'calc_terms_probs_ratio']
+    "get_phi",
+    "get_theta",
+    "get_relevant_terms",
+    "get_salient_terms",
+    "get_docs",
+    "get_top_docs",
+    "calc_terms_marg_probs",
+    "calc_topics_marg_probs",
+    "calc_terms_probs_ratio",
+]
 from warnings import warn
 from importlib.util import find_spec
 from typing import Union, Optional, Sequence, List
@@ -13,7 +18,7 @@ from numpy import ndarray, zeros, argsort, array, arange, vstack
 from numpy import log as nplog
 from pandas import concat, Series, DataFrame
 
-tomotopy_installed = find_spec('tomotopy')
+tomotopy_installed = find_spec("tomotopy")
 if tomotopy_installed:
     from tomotopy import (
         LDAModel as tomotopyLDA,
@@ -23,14 +28,15 @@ if tomotopy_installed:
         HDPModel as tomotopyHDP,
         PTModel as tomotopyPT,
         SLDAModel as tomotopySLDA,
-        GDMRModel as tomotopyGDMR)
+        GDMRModel as tomotopyGDMR,
+    )
 
-gensim_installed = find_spec('gensim')
+gensim_installed = find_spec("gensim")
 if gensim_installed:
     from gensim.models.ldamodel import LdaModel as gensimLDA
     from gensim.models.ldamulticore import LdaMulticore as gensimLDAMC
 
-bitermplus_installed = find_spec('bitermplus')
+bitermplus_installed = find_spec("bitermplus")
 if bitermplus_installed:
     from bitermplus._btm import BTM
 
@@ -38,12 +44,11 @@ if bitermplus_installed:
 def __warn_package_installation(package_name: str):
     warn(
         f'Please install "{package_name}" package to analyze its models.\n'
-        f'Run `pip install {package_name}` in the console.')
+        f"Run `pip install {package_name}` in the console."
+    )
 
 
-def get_phi(
-        model: object,
-        vocabulary: Optional[Sequence] = None) -> DataFrame:
+def get_phi(model: object, vocabulary: Optional[Sequence] = None) -> DataFrame:
     """Get words vs topics matrix (phi).
 
     Returns ``phi`` matrix of shape W x T, where W is the number of words,
@@ -65,7 +70,6 @@ def get_phi(
     phi = None
 
     if _is_tomotopy(model):
-
         # Topics vs words distributions
         twd = list(map(model.get_topic_word_dist, range(model.k)))
 
@@ -76,7 +80,6 @@ def get_phi(
         phi.index = list(model.used_vocabs)
 
     elif _is_gensim(model):
-
         phi = DataFrame(model.get_topics().T)
         if vocabulary:
             phi.index = vocabulary
@@ -85,8 +88,8 @@ def get_phi(
         phi = model.df_words_topics_
 
     if isinstance(phi, DataFrame):
-        phi.index.name = 'words'
-        phi.columns.name = 'topics'
+        phi.index.name = "words"
+        phi.columns.name = "topics"
 
     return phi
 
@@ -94,8 +97,15 @@ def get_phi(
 def _is_tomotopy(model: object) -> bool:
     if tomotopy_installed:
         tomotopy_models = [
-            tomotopyLDA, tomotopyLLDA, tomotopyCT, tomotopyDMR, tomotopyHDP,
-            tomotopyPT, tomotopySLDA, tomotopyGDMR]
+            tomotopyLDA,
+            tomotopyLLDA,
+            tomotopyCT,
+            tomotopyDMR,
+            tomotopyHDP,
+            tomotopyPT,
+            tomotopySLDA,
+            tomotopyGDMR,
+        ]
         return any(map(partial(isinstance, model), tomotopy_models))
 
     __warn_package_installation("tomotopy")
@@ -119,9 +129,7 @@ def _is_btmplus(model: object) -> bool:
     return False
 
 
-def get_theta(
-        model: object,
-        corpus: Optional[List] = None) -> DataFrame:
+def get_theta(model: object, corpus: Optional[List] = None) -> Optional[DataFrame]:
     """Get topics vs documents (theta) matrix.
 
     Returns theta matrix of shape T x D, where T is the number of topics,
@@ -132,7 +140,7 @@ def get_theta(
     model : object
         Topic model instance.
     corpus : Optional[List], optional
-        Corpus.
+        Corpus (must be specified for a `gensim` model).
 
     Returns
     -------
@@ -147,8 +155,7 @@ def get_theta(
 
     elif _is_gensim(model):
         if corpus is None:
-            raise ValueError(
-                '`corpus` must be supplied for a gensim model')
+            raise ValueError("`corpus` must be supplied for a gensim model")
         tdd = list(map(model.get_document_topics, corpus))
         theta = DataFrame(zeros((len(tdd), model.num_topics)))
         for doc_id, doc_topic in enumerate(tdd):
@@ -160,14 +167,13 @@ def get_theta(
         theta = DataFrame(model.matrix_topics_docs_)
 
     if isinstance(theta, DataFrame):
-        theta.index.name = 'topics'
-        theta.columns.name = 'docs'
+        theta.index.name = "topics"
+        theta.columns.name = "docs"
 
     return theta
 
 
-def get_docs(
-        model: object) -> List[str]:
+def get_docs(model: object) -> Optional[List[str]]:
     """Retrieve documents from topic model object.
 
     Parameters
@@ -183,19 +189,19 @@ def get_docs(
     if _is_tomotopy(model):
         docs_raw = map(lambda x: x.words, model.docs)
         return list(
-            map(
-                lambda doc: " ".join(map(lambda x: model.vocabs[x], doc)),
-                docs_raw))
+            map(lambda doc: " ".join(map(lambda x: model.vocabs[x], doc)), docs_raw)
+        )
     return None
 
 
 def get_top_docs(
-        docs: Sequence[str],
-        model: object = None,
-        theta: ndarray = None,
-        corpus: Optional[List] = None,
-        docs_num: int = 5,
-        topics: Sequence[int] = None) -> DataFrame:
+    docs: Sequence[str],
+    model: object = None,
+    theta: Optional[ndarray] = None,
+    corpus: Optional[List] = None,
+    docs_num: int = 5,
+    topics: Optional[Sequence[int]] = None,
+) -> DataFrame:
     """Get top documents for all (or a selected) topic.
 
     Parameters
@@ -231,20 +237,19 @@ def get_top_docs(
 
     def _select_docs(docs, theta, topic_id: int):
         probs = theta[topic_id, :]
-        idx = argsort(probs)[:-docs_num-1:-1]
+        idx = argsort(probs)[: -docs_num - 1 : -1]
         result = Series(list(map(lambda x: docs[x], idx)))
-        result.name = f'topic{topic_id}'
+        result.name = f"topic{topic_id}"
         return result
 
     topics_num = theta.shape[0]
     topics_idx = arange(topics_num) if topics is None else topics
-    return concat(
-        map(lambda x: _select_docs(docs, theta, x), topics_idx), axis=1)
+    return concat(map(lambda x: _select_docs(docs, theta, x), topics_idx), axis=1)
 
 
 def calc_topics_marg_probs(
-        theta: Union[DataFrame, ndarray],
-        topic_id: int = None) -> Union[DataFrame, ndarray]:
+    theta: Union[DataFrame, ndarray], topic_id: Optional[int] = None
+) -> ndarray:
     """Calculate marginal topics probabilities.
 
     Parameters
@@ -259,18 +264,18 @@ def calc_topics_marg_probs(
     Union[pandas.DataFrame, numpy.ndarray]
         Marginal topics probabilities.
     """
+    p_t = array(theta).sum(axis=1)
+    p_t /= p_t.sum()
     if topic_id is not None:
-        if isinstance(theta, ndarray):
-            return theta[topic_id, :].sum()
-        if isinstance(theta, DataFrame):
-            return theta.iloc[topic_id, :].sum()
-
-    return theta.sum(axis=1)
+        return p_t[topic_id]
+    return p_t
 
 
 def calc_terms_marg_probs(
-        phi: Union[ndarray, DataFrame],
-        word_id: Optional[int] = None) -> Union[ndarray, Series]:
+    phi: Union[ndarray, DataFrame],
+    p_t: Union[ndarray, Series],
+    word_id: Optional[int] = None,
+) -> ndarray:
     """Calculate marginal terms probabilities.
 
     Parameters
@@ -285,19 +290,13 @@ def calc_terms_marg_probs(
     Union[numpy.ndarray, pandas.Series]
         Marginal terms probabilities.
     """
+    p_w = (array(phi) * array(p_t)).sum(axis=1)
     if word_id is not None:
-        if isinstance(phi, ndarray):
-            return phi[word_id, :].sum()
-        if isinstance(phi, DataFrame):
-            return phi.iloc[word_id, :].sum()
-
-    return phi.sum(axis=1)
+        return p_w[word_id]
+    return p_w
 
 
-def get_salient_terms(
-        terms_freqs: ndarray,
-        phi: ndarray,
-        theta: ndarray) -> ndarray:
+def get_salient_terms(phi: ndarray, theta: ndarray) -> ndarray:
     """Get salient terms.
 
     Calculated as:
@@ -306,8 +305,6 @@ def get_salient_terms(
 
     Parameters
     ----------
-    terms_freqs : numpy.ndarray
-        Words frequencies.
     phi : numpy.ndarray
         Words vs topics matrix.
     theta : numpy.ndarray
@@ -318,18 +315,24 @@ def get_salient_terms(
     numpy.ndarray
         Terms saliency values.
     """
-    p_t = array(calc_topics_marg_probs(theta))
-    p_w = array(calc_terms_marg_probs(phi))
+    p_t = calc_topics_marg_probs(theta)
+    p_w = calc_terms_marg_probs(phi, p_t)
 
     def _p_tw(phi, w, t):
-        return phi[w, t] * p_t[t] / p_w[w]
-
-    saliency = array((
-        terms_freqs[w] * sum((
-            _p_tw(phi, w, t) * log(_p_tw(phi, w, t) / p_t[t])
-            for t in range(phi.shape[1])))
-        for w in range(phi.shape[0])
-    ))
+        return array(phi)[w, t] * p_t[t] / p_w[w]
+
+    saliency = array(
+        [
+            p_w[w]
+            * sum(
+                (
+                    _p_tw(phi, w, t) * log(_p_tw(phi, w, t) / p_t[t])
+                    for t in range(phi.shape[1])
+                )
+            )
+            for w in range(phi.shape[0])
+        ]
+    )
     # saliency(term w) = frequency(w)
     # * [sum_t p(t | w) * log(p(t | w)/p(t))] for topics t
     # p(t | w) = p(w | t) * p(t) / p(w)
@@ -337,10 +340,8 @@ def get_salient_terms(
 
 
 def calc_terms_probs_ratio(
-        phi: DataFrame,
-        topic: int,
-        terms_num: int = 30,
-        lambda_: float = 0.6) -> DataFrame:
+    phi: DataFrame, topic: int, terms_num: int = 30, lambda_: float = 0.6
+) -> DataFrame:
     """Get terms conditional and marginal probabilities.
 
     Parameters
@@ -368,35 +369,38 @@ def calc_terms_probs_ratio(
     pandas.DataFrame
         Words conditional and marginal probabilities.
     """
-    p_cond_name = 'Conditional term probability, p(w | t)'
-    p_cond = phi.iloc[:, topic]\
-        .rename(p_cond_name)\
-        if isinstance(phi, DataFrame)\
+    p_cond_name = "Conditional term probability, p(w | t)"
+    p_cond = (
+        phi.iloc[:, topic].rename(p_cond_name)
+        if isinstance(phi, DataFrame)
         else Series(phi[:, topic], name=p_cond_name)
+    )
 
-    p_marg_name = 'Marginal term probability, p(w)'
-    p_marg = phi.sum(axis=1)\
-        .rename(p_marg_name)\
-        if isinstance(phi, DataFrame)\
+    p_marg_name = "Marginal term probability, p(w)"
+    p_marg = (
+        phi.sum(axis=1).rename(p_marg_name)
+        if isinstance(phi, DataFrame)
         else Series(phi[:, topic], name=p_marg_name)
+    )
 
     terms_probs = concat((p_marg, p_cond), axis=1)
     relevant_idx = get_relevant_terms(phi, topic, lambda_).index
     terms_probs_slice = terms_probs.loc[relevant_idx].head(terms_num)
 
-    return terms_probs_slice\
-        .reset_index(drop=False)\
+    return (
+        terms_probs_slice.reset_index(drop=False)
         .melt(
             id_vars=[terms_probs_slice.index.name],
-            var_name='Type',
-            value_name='Probability')\
-        .rename(columns={terms_probs_slice.index.name: 'Terms'})
+            var_name="Type",
+            value_name="Probability",
+        )
+        .rename(columns={terms_probs_slice.index.name: "Terms"})
+    )
 
 
 def get_relevant_terms(
-        phi: Union[ndarray, DataFrame],
-        topic: int,
-        lambda_: float = 0.6) -> Series:
+    phi: Union[ndarray, DataFrame], topic: int, lambda_: float = 0.6
+) -> Series:
     """Select relevant terms.
 
     Parameters
@@ -422,11 +426,10 @@ def get_relevant_terms(
     pandas.Series
         Terms sorted by relevance (descendingly).
     """
-    phi_topic = phi.iloc[:, topic]\
-        if isinstance(phi, DataFrame)\
-        else phi[:, topic]
+    phi_topic = phi.iloc[:, topic] if isinstance(phi, DataFrame) else phi[:, topic]
 
     # relevance = lambda * log(p(w | t)) + (1 - lambda) * log(p(w | t) / p(w))
-    relevance = lambda_ * nplog(phi_topic)\
-        + (1 - lambda_) * nplog(phi_topic / phi.sum(axis=1))
+    relevance = lambda_ * nplog(phi_topic) + (1 - lambda_) * nplog(
+        phi_topic / phi.sum(axis=1)
+    )
     return relevance.sort_values(ascending=False)