wordlevel 0.0.1a2__py3-none-any.whl

wordlevel/__init__.py

@@ -0,0 +1,11 @@
+from wordlevel.api.catalog.catalog import Catalog
+from wordlevel.api.comparison.comparison import comp
+from wordlevel.api.lexicons.lexicon import lex
+from wordlevel.datasets.dataset import Dataset
+
+try:
+    from wordlevel._version import __version__
+except ImportError:
+    __version__ = "0.0.0"
+
+__all__ = ["__version__", "Catalog", "comp", "Dataset", "lex"]

wordlevel/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.0.1a2'
+__version_tuple__ = version_tuple = (0, 0, 1, 'a2')
+
+__commit_id__ = commit_id = None

wordlevel/api/catalog/catalog.py

@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+from collections.abc import Collection, Mapping, Sequence
+from functools import cached_property
+from types import MappingProxyType
+
+import polars as pl
+
+from wordlevel.api.catalog.constructors import (
+    from_catalog,
+    from_polars,
+    from_dict_of_freqs,
+    from_list_of_freqs,
+)
+from wordlevel.api.catalog.plot import Plotter
+from wordlevel.api.catalog.constructors import CatalogData, DictOfFreqs, ListOfFreqs
+from wordlevel.api.comparison.comparison import Comparison, ComparisonResult
+
+
+class Catalog:
+    """
+    The workspace for corpora and making comparisons between them.
+
+    For specific construction functions, see the [construction](construct/index.md) documentation.
+
+    Args:
+        data:
+            The corpora token-frequency data. Accepts a Polars DataFrame or LazyFrame,
+            [`ListOfFreqs`][wordlevel.api.catalog.constructors.ListOfFreqs],
+            [`DictOfFreqs`][wordlevel.api.catalog.constructors.DictOfFreqs], or an existing
+            `Catalog`. The class will attempt to infer the structure of the data and use the
+            appropriate constructor.
+        token_col:
+            If the `data` is a Polars dataframe, the column that contains the tokens (words,
+            subwords, n-grams, etc.). Ignored otherwise
+        corpora:
+            If the `data` is a [`ListOfFreqs`][wordlevel.api.catalog.constructors.ListOfFreqs], the
+            names of the corpora, listed in the same order as `data`. If not provided, will default
+            to `corpus1`, `corpus2`, etc. If the `data` is a Polars Dataframe or Lazyframe, the
+            names of the columns containing token frequencies, which must be provided. Ignored
+            otherwise
+
+    Raises:
+        TypeError: If `data` is not one of the supported types.
+    """
+
+    def __init__(
+        self,
+        data: pl.DataFrame | pl.LazyFrame | Catalog | CatalogData | DictOfFreqs | ListOfFreqs,
+        token_col: str | None = None,
+        corpora: Sequence[str] | None = None,
+    ) -> None:
+        if isinstance(data, CatalogData):
+            init_data = data
+        elif isinstance(data, Catalog):
+            init_data = from_catalog(data)
+        elif isinstance(data, (pl.DataFrame, pl.LazyFrame)):
+            if token_col is None:
+                raise ValueError("`token_col` must be specified if providing a DataFrame")
+            if corpora is None:
+                raise ValueError("`corpora` must be specified if providing a DataFrame")
+            init_data = from_polars(
+                df=data,
+                token_col=token_col,
+                freq_cols=corpora,
+            )
+        elif isinstance(data, Mapping):
+            init_data = from_dict_of_freqs(freqs=data)
+        elif isinstance(data, Sequence):
+            init_data = from_list_of_freqs(freqs=data, corpora=corpora)
+        else:
+            raise TypeError(f"Input type not supported: {type(data)}")
+
+        self._init_from_catalog_data(init_data)
+
+    def _init_from_catalog_data(self, data: CatalogData) -> None:
+        """
+        Initializes the catalog given the core `CatalogData`.
+        """
+        self._df = data.df
+        self._token_col = data.token_col
+        self._freq_cols = set(data.freq_cols)
+        self._normalization_suffix = data.normalization_suffix
+        self._comparisons = data.comparisons
+
+    @classmethod
+    def from_catalog(cls, catalog: Catalog) -> Catalog:
+        """
+        Creates a new catalog from an existing one.
+
+        The new catalog copies the data of the original one without mutating it.
+
+        Example:
+            ```python
+            --8<-- "snippets/catalog/catalog.py:from_catalog"
+            ```
+
+        Args:
+            catalog: An existing `Catalog`
+
+        Returns:
+            A new `Catalog` with the same internal data as the provided catalog
+        """
+        init_data = from_catalog(catalog)
+
+        return cls(data=init_data)
+
+    @classmethod
+    def from_polars(
+        cls,
+        df: pl.DataFrame | pl.LazyFrame,
+        token_col: str,
+        corpora: Collection[str],
+        normalize: bool = True,
+    ) -> Catalog:
+        """
+        Creates a `Catalog` of corpora from a Polars dataframe.
+
+        Example:
+            ```python
+            --8<-- "snippets/catalog/catalog.py:from_polars"
+            ```
+
+        Args:
+            df:
+                A Polars DataFrame or LazyFrame containing corpus information. Rows are tokens and
+                columns are their frequencies in different corpora
+            token_col:
+                The column that contains the tokens (words, subwords, n-grams, etc.)
+            corpora:
+                The columns that include the token frequencies for each corpus
+            normalize:
+                Whether to normalize the corpus frequencies for each corpus so that they are in the
+                range $[0, 1]$ and sum to 1. The original corpus frequencies will be preserved
+
+        Returns:
+            A `Catalog` of corpora representing the provided tokens and frequencies
+
+        Raises:
+            KeyError: If `token_col` is not a column in `df`.
+            KeyError: If any column in `corpora` is not a column in `df`.
+            ValueError: If fewer than two columns are provided in `corpora`.
+        """
+        init_data = from_polars(
+            df=df,
+            token_col=token_col,
+            freq_cols=corpora,
+            normalize=normalize,
+        )
+
+        return cls(data=init_data)
+
+    @classmethod
+    def from_list_of_freqs(
+        cls,
+        freqs: ListOfFreqs,
+        corpora: Sequence[str] | None = None,
+    ) -> Catalog:
+        """
+        Creates a `Catalog` of corpora from a sequence of token-to-frequency mappings, each
+        representing the frequencies of tokens in a particular corpus.
+
+        Example:
+            ```python
+            --8<-- "snippets/catalog/catalog.py:from_list_of_freqs"
+            ```
+
+        Args:
+            freqs:
+                A sequence of token-to-frequency mappings, one per corpus. Each mapping is
+                `{token: frequency}` for a single corpus
+            corpora:
+                The names of the corpora, listed in the same order as `freqs`. If not provided, will
+                default to `corpus1`, `corpus2`, etc.
+
+        Returns:
+            A `Catalog` of corpora representing the provided tokens and frequencies
+
+        Raises:
+            ValueError: If `freqs` and `corpora` have different lengths.
+            ValueError: If fewer than two corpora are provided in `freqs`.
+            ValueError: If any corpus in `freqs` has no token frequencies.
+        """
+        init_data = from_list_of_freqs(freqs=freqs, corpora=corpora)
+
+        return cls(data=init_data)
+
+    @classmethod
+    def from_dict_of_freqs(cls, freqs: DictOfFreqs) -> Catalog:
+        """
+        Creates a `Catalog` of corpora from named token-to-frequency mappings.
+
+        Example:
+            ```python
+            --8<-- "snippets/catalog/catalog.py:from_dict_of_freqs"
+            ```
+
+        Args:
+            freqs:
+                A mapping of corpus names to token-to-frequency mappings, one per corpus. Each
+                mapping is `{corpus: {token: frequency}}`, with one mapping per corpus
+
+        Returns:
+            A `Catalog` of corpora representing the provided tokens and frequencies
+
+        Raises:
+            ValueError: If fewer than two corpora are provided in `freqs`.
+            ValueError: If any corpus in `freqs` has no token frequencies.
+        """
+        init_data = from_dict_of_freqs(freqs=freqs)
+
+        return cls(data=init_data)
+
+    @property
+    def comparisons(self) -> Mapping[str, ComparisonResult]:
+        """
+        Comparisons made between corpora in the catalog, keyed by each comparison's alias.
+        """
+        return MappingProxyType(self._comparisons)
+
+    @property
+    def df(self) -> pl.DataFrame:
+        """
+        Dataframe representing the frequencies of tokens in the corpora in the catalog.
+        """
+        return self._df.collect()
+
+    @cached_property
+    def plot(self) -> Plotter:
+        """
+        Namespace for plotting methods. See [available plots](plot/index.md).
+        """
+        return Plotter(self)
+
+    def with_comparisons(self, *comparisons: Comparison) -> Catalog:
+        """
+        Returns a new catalog with comparisons added to it.
+
+        New comparisons will overwrite existing comparisons with the same name.
+
+        Example:
+            ```python
+            --8<-- "snippets/catalog/catalog.py:setup"
+
+            --8<-- "snippets/catalog/catalog.py:with_comparisons"
+            ```
+
+        Args:
+            comparisons: The comparisons to add to the catalog
+
+        Returns:
+            The catalog with the comparisons added to it
+        """
+        catalog = self.__class__(self)
+
+        for comparison in comparisons:
+            result = ComparisonResult.from_comparison(
+                comparison,
+                self._df,
+                token_col=self._token_col,
+                normalization_suffix=self._normalization_suffix,
+            )
+            catalog._comparisons[result.alias] = result
+
+        return catalog

wordlevel/api/catalog/constructors.py

@@ -0,0 +1,287 @@
+from __future__ import annotations
+
+from collections.abc import Collection
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterable, Mapping, Sequence, TypeAlias
+
+import polars as pl
+from polars import col
+
+from wordlevel.constants import NORMALIZATION_SUFFIX, TOKEN
+from wordlevel.core.utils import normalize_freqs
+
+
+if TYPE_CHECKING:
+    from wordlevel.api.catalog.catalog import Catalog
+    from wordlevel.api.comparison.comparison import ComparisonResult
+
+
+@dataclass
+class CatalogData:
+    """
+    The core data that is required to initialize and copy a `Catalog`
+
+    Args:
+        df: Polars
+            DataFrame representing the frequencies of tokens in the corpora in the catalog.
+        token_col:
+            The column in `df` that contains the tokens (words, subwords, n-grams, etc.)
+        freq_cols:
+            The columns in `df` that include the token frequencies for each corpus
+        normalization_suffix:
+            The suffix used on the `freq_cols` to indicate their normalized forms where all
+            frequencies are in the range $[0, 1]$ and sum to 1
+        comparisons:
+            Comparisons made between corpora in the catalog, keyed by each comparison's alias
+    """
+
+    df: pl.LazyFrame
+    token_col: str
+    freq_cols: Collection[str]
+    normalization_suffix: str
+    comparisons: dict[str, ComparisonResult]
+
+
+ListOfFreqs: TypeAlias = Sequence[Mapping[str, float]]
+"""
+A sequence of token-to-frequency mappings, one per corpus. Each mapping is `{word: frequency}` for 
+a single corpus
+
+Example:
+    ```python
+    list_of_freqs = [
+        {"hi": 1, "hey": 2, "hello": 3},
+        {"hi": 2, "hey": 4, "bye": 6}
+    ]
+    ```
+"""
+
+
+DictOfFreqs: TypeAlias = Mapping[str, Mapping[str, float]]
+"""
+A mapping of corpus names to word-to-frequency mappings. The mapping is 
+`{corpus: {word: frequency}}`
+
+Example:
+    ```python
+    dict_of_freqs = {
+        "corpus1": {"hi": 1, "hey": 2, "hello": 3},
+        "corpus2": {"hi": 2, "hey": 4, "bye": 6}
+    }
+    ```
+"""
+
+
+def from_catalog(catalog: Catalog) -> CatalogData:
+    """
+    Creates the data for a new catalog from an existing one.
+
+    Copies the data of the original catalog without mutating it.
+
+    Args:
+        catalog: An existing `Catalog`
+
+    Returns:
+        The `CatalogData` needed to initialize a new catalog
+    """
+    return CatalogData(
+        df=catalog._df.clone().lazy(),
+        token_col=catalog._token_col,
+        freq_cols=catalog._freq_cols,
+        normalization_suffix=catalog._normalization_suffix,
+        comparisons=catalog._comparisons.copy(),
+    )
+
+
+def from_polars(
+    df: pl.DataFrame | pl.LazyFrame,
+    token_col: str,
+    freq_cols: Collection[str],
+    normalize: bool = True,
+) -> CatalogData:
+    """
+    Creates the data for a new catalog from a Polars dataframe.
+
+    Args:
+        df:
+            A Polars DataFrame or LazyFrame containing corpus information. Rows are tokens and
+            columns are their frequencies in different corpora
+        token_col:
+            The column that contains the tokens (words, subwords, n-grams, etc.)
+        freq_cols:
+            The columns that include the token frequencies for each corpus
+        normalize:
+            Whether to normalize the corpus frequencies for each corpus so that they are in the
+            range $[0, 1]$ and sum to 1. The original corpus frequencies will be preserved
+
+    Returns:
+        The `CatalogData` needed to initialize a new catalog
+
+    Raises:
+        KeyError: If `token_col` is not a column in `df`.
+        KeyError: If any column in `freq_cols` is not a column in `df`.
+        ValueError: If fewer than two columns are provided in `freq_cols`.
+    """
+    # Check all columns exist
+    deduped_freq_cols = set(freq_cols)
+
+    if isinstance(df, pl.DataFrame):
+        cols = set(df.columns)
+    else:
+        cols = set(df.collect_schema().names())
+
+    if token_col not in cols:
+        raise KeyError(f"Token column not in provided dataframe: {token_col}")
+
+    if len(deduped_freq_cols) < 2:
+        raise ValueError("At least two frequency columns must be provided via `freq_cols`")
+
+    for freq_col in deduped_freq_cols:
+        if freq_col not in cols:
+            raise KeyError(f"Frequency column not in provided dataframe: {freq_col}")
+
+    # Do necessary transformations
+    lf = df.clone().lazy()
+    lf = lf.select(token_col, *[col(freq_col).fill_null(0) for freq_col in deduped_freq_cols])
+
+    if normalize:
+        lf = _normalize_frequency_columns(
+            lf,
+            freq_cols=deduped_freq_cols,
+            normalization_suffix=NORMALIZATION_SUFFIX,
+        )
+        normalization_suffix = NORMALIZATION_SUFFIX
+    else:
+        normalization_suffix = ""
+
+    return CatalogData(
+        df=lf,
+        token_col=token_col,
+        freq_cols=deduped_freq_cols,
+        normalization_suffix=normalization_suffix,
+        comparisons=dict(),
+    )
+
+
+def from_list_of_freqs(
+    freqs: ListOfFreqs,
+    corpora: Sequence[str] | None = None,
+) -> CatalogData:
+    """
+    Creates the data for a new catalog from a sequence of token-to-frequency mappings, each
+    representing the frequencies of tokens in a particular corpus.
+
+    Args:
+        freqs:
+            A sequence of token-to-frequency mappings, one per corpus. Each mapping is
+            `{token: frequency}` for a single corpus
+        corpora:
+            The names of the corpora, listed in the same order as `freqs`. If not provided, will
+            default to `corpus1`, `corpus2`, etc.
+
+    Returns:
+        The `CatalogData` needed to initialize a new catalog
+
+    Raises:
+        ValueError: If `freqs` and `corpora` have different lengths.
+        ValueError: If fewer than two corpora are provided in `freqs`.
+        ValueError: If any corpus in `freqs` has no token frequencies.
+    """
+    if corpora is None:
+        corpora = [f"corpus{n}" for n in range(1, len(freqs) + 1)]
+    else:
+        n_freqs = len(freqs)
+        n_corpora = len(corpora)
+        if n_freqs != n_corpora:
+            raise ValueError(
+                f"`freqs` and `corpora` must be the same length ({n_freqs} vs. {n_corpora})"
+            )
+
+    if len(freqs) < 2:
+        raise ValueError("At least two frequency sets (corpora) need to be provided")
+
+    return _from_freqs(
+        freqs=zip(corpora, freqs),
+        token=TOKEN,
+    )
+
+
+def from_dict_of_freqs(
+    freqs: DictOfFreqs,
+) -> CatalogData:
+    """
+    Creates the data for a new catalog from named token-to-frequency mappings.
+
+    Args:
+        freqs:
+            A mapping of corpus names to token-to-frequency mappings, one per corpus. Each
+            mapping is `{corpus: {token: frequency}}`, with one mapping per corpus
+
+    Returns:
+        The `CatalogData` needed to initialize a new catalog
+
+    Raises:
+        ValueError: If fewer than two corpora are provided in `freqs`.
+        ValueError: If any corpus in `freqs` has no token frequencies.
+    """
+    if len(freqs) < 2:
+        raise ValueError("At least two frequency sets (corpora) need to be provided")
+
+    return _from_freqs(
+        freqs=freqs.items(),
+        token=TOKEN,
+    )
+
+
+def _from_freqs(
+    freqs: Iterable[tuple[str, Mapping[str, float]]],
+    token: str = TOKEN,
+) -> CatalogData:
+    """
+    Creates the data for a new catalog from token-frequency data structures
+
+    Args:
+        freqs:
+            A sequence of tuples, one per corpus. Each tuple is `(corpus, {token: frequency})`
+        token:
+            The name the use for the column representing tokens in the produced catalog dataframe
+
+    Returns:
+        The `CatalogData` needed to initialize a new catalog
+    """
+    freq_cols: list[str] = []
+    lf = pl.LazyFrame(schema={token: pl.String})
+
+    for corpus, token2freq in freqs:
+        if not token2freq:
+            raise ValueError(f"There are no token frequencies for `{corpus}`")
+
+        freq_cols.append(corpus)
+
+        tokens, token_freqs = zip(*token2freq.items())
+        lf_corpus = pl.LazyFrame([tokens, token_freqs], schema=[token, corpus])
+        lf = lf.join(lf_corpus, on=token, how="full", coalesce=True)
+
+    return from_polars(
+        lf,
+        token_col=token,
+        freq_cols=freq_cols,
+    )
+
+
+def _normalize_frequency_columns(
+    df: pl.LazyFrame,
+    freq_cols: Collection[str],
+    normalization_suffix: str = NORMALIZATION_SUFFIX,
+) -> pl.LazyFrame:
+    """
+    Normalizes columns in a dataframe so all values are in the range $[0, 1]$ and sum to 1. The
+    normalized columns are named using a `normalization_suffix`, appended to the original column
+    names.
+    """
+    normed_cols = [
+        normalize_freqs(col(freq_col)).alias(f"{freq_col}{normalization_suffix}")
+        for freq_col in freq_cols
+    ]
+
+    return df.with_columns(*normed_cols)