phoneme-entropy 0.1.0__tar.gz

phoneme_entropy-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install (with dev extras)
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest -q
+
+  build:
+    # Confirm the sdist/wheel build and pass metadata checks on every push.
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build & check distribution
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*

phoneme_entropy-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,86 @@
+name: Release
+
+# Two-stage publishing, both via PyPI Trusted Publishing (OIDC — no stored tokens):
+#
+#   * push a tag  v*             -> build + publish to TestPyPI   (release candidate)
+#   * publish a GitHub Release   -> build + publish to PyPI       (promote to real PyPI)
+#   * manual dispatch            -> build + publish to TestPyPI
+#
+# Typical flow:
+#   git tag v0.1.0 && git push origin v0.1.0     # -> lands on TestPyPI, smoke-test it
+#   # then, on GitHub, cut a Release for tag v0.1.0  -> lands on PyPI
+#
+# One-time setup (per index) at https://<index>/manage/account/publishing/ :
+#   add a pending publisher -> owner: suchirsalhan, repo: phoneme,
+#   workflow: release.yml, environment: testpypi (on TestPyPI) / pypi (on PyPI).
+# Also create matching GitHub environments (repo Settings -> Environments):
+#   `testpypi` and `pypi`.
+#
+# Prefer API tokens? Remove the environment/permissions blocks, add repo secrets
+# TEST_PYPI_API_TOKEN / PYPI_API_TOKEN, and set `password:` on the publish steps.
+
+on:
+  push:
+    tags: ["v*"]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build sdist + wheel
+        run: |
+          python -m pip install --upgrade pip build twine
+          python -m build
+          twine check dist/*
+      - name: Upload dist artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish-testpypi:
+    # Release candidates: tag pushes and manual runs go to TestPyPI.
+    needs: build
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write        # required for Trusted Publishing (OIDC)
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          # password: ${{ secrets.TEST_PYPI_API_TOKEN }}   # token auth alternative
+
+  publish-pypi:
+    # Promotion: publishing a GitHub Release ships to real PyPI.
+    needs: build
+    if: github.event_name == 'release'
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write        # required for Trusted Publishing (OIDC)
+    steps:
+      - name: Download dist artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        # repository-url defaults to https://upload.pypi.org/legacy/ (real PyPI)
+        # with:
+        #   password: ${{ secrets.PYPI_API_TOKEN }}         # token auth alternative

phoneme_entropy-0.1.0/.gitignore

@@ -0,0 +1,45 @@
+# --- Local, developer-only instructions (never commit) ---
+LOCAL_INSTRUCTIONS.md
+
+# --- Python ---
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+.eggs/
+build/
+dist/
+wheels/
+*.egg
+.installed.cfg
+
+# --- Virtual environments ---
+.venv/
+venv/
+env/
+ENV/
+.python-version
+
+# --- Testing / coverage ---
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.ruff_cache/
+.mypy_cache/
+
+# --- Jupyter ---
+.ipynb_checkpoints/
+
+# --- Hugging Face / data caches (do not commit private data) ---
+.cache/
+*.arrow
+hf_cache/
+data_cache/
+
+# --- OS / editor ---
+.DS_Store
+.idea/
+.vscode/
+*.swp

phoneme_entropy-0.1.0/CITATION.cff

@@ -0,0 +1,21 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below, along with the upstream entropy-estimators project."
+title: "phoneme-entropy: phoneme-level entropy, informativity, and maximum-entropy tools"
+authors:
+  - family-names: "Moscoso del Prado Martín"
+    given-names: Fermín
+  - family-names: Salhan
+    given-names: Suchir
+type: software
+license: MIT
+repository-code: "https://github.com/suchirsalhan/phoneme"
+version: "0.1.0"
+references:
+  - type: software
+    title: "entropy-estimators"
+    authors:
+      - family-names: "Moscoso del Prado Martín"
+        given-names: "Fermín"
+    repository-code: "https://github.com/fermosc24/entropy-estimators"
+    license: MIT
+    year: 2025

phoneme_entropy-0.1.0/LICENSE

@@ -0,0 +1,25 @@
+MIT License
+
+Copyright (c) 2026 Fermín Moscoso del Prado Martín & Suchir Salhan
+
+Portions of this library extend and depend on the `entropy-estimators` project
+(https://github.com/fermosc24/entropy-estimators), Copyright (c) 2025 Fermín
+Moscoso del Prado Martín, also distributed under the MIT License.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

phoneme_entropy-0.1.0/PKG-INFO

@@ -0,0 +1,111 @@
+Metadata-Version: 2.4
+Name: phoneme-entropy
+Version: 0.1.0
+Summary: Phoneme-level entropy, informativity, and maximum-entropy tools, extending entropy-estimators.
+Project-URL: Homepage, https://github.com/suchirsalhan/phoneme
+Project-URL: Repository, https://github.com/suchirsalhan/phoneme
+Project-URL: Upstream (entropy-estimators), https://github.com/fermosc24/entropy-estimators
+Author: Fermín Moscoso del Prado Martín
+Author-email: Suchir Salhan <suchirsalhan@gmail.com>
+License: MIT
+License-File: LICENSE
+Keywords: dirichlet,entropy,information theory,maximum entropy,phonemes,phonology,psycholinguistics
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.9
+Requires-Dist: mpmath>=1.2
+Requires-Dist: numpy>=1.21
+Requires-Dist: pandas>=1.3
+Requires-Dist: scipy>=1.7
+Provides-Extra: data
+Requires-Dist: datasets>=2.14; extra == 'data'
+Requires-Dist: huggingface-hub>=0.16; extra == 'data'
+Provides-Extra: dev
+Requires-Dist: jupyter; extra == 'dev'
+Requires-Dist: matplotlib>=3.4; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Requires-Dist: ruff>=0.1; extra == 'dev'
+Requires-Dist: seaborn>=0.11; extra == 'dev'
+Provides-Extra: plot
+Requires-Dist: matplotlib>=3.4; extra == 'plot'
+Requires-Dist: seaborn>=0.11; extra == 'plot'
+Description-Content-Type: text/markdown
+
+# phoneme-entropy
+
+Phoneme-level entropy, informativity, and maximum-entropy tools. A standalone
+library that bundles the entropy estimators from
+[`entropy-estimators`](https://github.com/fermosc24/entropy-estimators) (MIT,
+© 2025 fermosc24) and adds phonology-oriented analysis utilities.
+
+## Install
+
+```bash
+pip install phoneme-entropy
+# or, from source:
+pip install .
+```
+
+Optional extras: `pip install "phoneme-entropy[plot]"` (matplotlib/seaborn for
+`plot_ranks`), `pip install "phoneme-entropy[data]"` (Hugging Face `datasets`).
+
+## Usage
+
+```python
+from collections import defaultdict
+import numpy as np, pandas as pd
+from phoneme_entropy import (
+    Entropy, estimate_alpha_entropy,
+    segment_informativity, phoneme_prefix_entropy,
+    compute_maxent_from_matrix,
+)
+from phoneme_entropy.data import load_sample
+
+sample = load_sample()                       # bundled CELEX sample (Word = space-separated phonemes)
+
+# Per-phoneme next-phoneme surprisal and prefix entropy
+surprisal = segment_informativity(sample["Word"])
+prefix_H  = phoneme_prefix_entropy(sample["Word"])
+
+# Symmetric-Dirichlet concentration implied by an entropy estimate
+phon = defaultdict(int)
+for w in sample["Word"]:
+    for p in w.split(): phon[p] += 1
+alpha, se = estimate_alpha_entropy(list(phon.values()), n_boot=0)
+```
+
+See [`examples/quickstart.py`](examples/quickstart.py) and
+[`examples/usage_example.ipynb`](examples/usage_example.ipynb) for a full
+walkthrough.
+
+### Data
+
+A small CELEX-derived sample ships with the package (`load_sample()`). The full
+sample is mirrored as a **gated, permissions-only** Hugging Face dataset
+[`suchirsalhan/celex-en-phoneme-sample`](https://huggingface.co/datasets/suchirsalhan/celex-en-phoneme-sample)
+— access is granted to holders of a valid CELEX/LDC license
+([LDC96L14](https://catalog.ldc.upenn.edu/docs/LDC96L14/celex.readme.html)):
+
+```python
+from phoneme_entropy.data import load_hf_dataset
+ds = load_hf_dataset(split="train")   # after your access request is approved
+```
+
+## API
+
+| Import | Purpose |
+| --- | --- |
+| `Entropy`, `FreqShrink`, `JS_KullbackLeibler`, `JS_JensenShannon` | Entropy / divergence estimators (MLE, JSE, Chao-Shen, CWJ, NBRS, NSB), vendored. |
+| `estimate_alpha_entropy`, `dirichlet_entropy`, `plot_ranks` | Symmetric-Dirichlet inference and rank/probability curves. |
+| `segment_informativity`, `phoneme_prefix_entropy` | Segmental informativity over phoneme strings. |
+| `compute_maxent_from_matrix` | Maximum-entropy fit to feature expectations. |
+| `phoneme_entropy.data.load_sample`, `load_hf_dataset` | Data loaders. |
+
+## License
+
+MIT — see [`LICENSE`](LICENSE). Bundles third-party code; see
+[`THIRD_PARTY_LICENSES`](THIRD_PARTY_LICENSES). Please cite via
+[`CITATION.cff`](CITATION.cff).

phoneme_entropy-0.1.0/README.md

@@ -0,0 +1,75 @@
+# phoneme-entropy
+
+Phoneme-level entropy, informativity, and maximum-entropy tools. A standalone
+library that bundles the entropy estimators from
+[`entropy-estimators`](https://github.com/fermosc24/entropy-estimators) (MIT,
+© 2025 fermosc24) and adds phonology-oriented analysis utilities.
+
+## Install
+
+```bash
+pip install phoneme-entropy
+# or, from source:
+pip install .
+```
+
+Optional extras: `pip install "phoneme-entropy[plot]"` (matplotlib/seaborn for
+`plot_ranks`), `pip install "phoneme-entropy[data]"` (Hugging Face `datasets`).
+
+## Usage
+
+```python
+from collections import defaultdict
+import numpy as np, pandas as pd
+from phoneme_entropy import (
+    Entropy, estimate_alpha_entropy,
+    segment_informativity, phoneme_prefix_entropy,
+    compute_maxent_from_matrix,
+)
+from phoneme_entropy.data import load_sample
+
+sample = load_sample()                       # bundled CELEX sample (Word = space-separated phonemes)
+
+# Per-phoneme next-phoneme surprisal and prefix entropy
+surprisal = segment_informativity(sample["Word"])
+prefix_H  = phoneme_prefix_entropy(sample["Word"])
+
+# Symmetric-Dirichlet concentration implied by an entropy estimate
+phon = defaultdict(int)
+for w in sample["Word"]:
+    for p in w.split(): phon[p] += 1
+alpha, se = estimate_alpha_entropy(list(phon.values()), n_boot=0)
+```
+
+See [`examples/quickstart.py`](examples/quickstart.py) and
+[`examples/usage_example.ipynb`](examples/usage_example.ipynb) for a full
+walkthrough.
+
+### Data
+
+A small CELEX-derived sample ships with the package (`load_sample()`). The full
+sample is mirrored as a **gated, permissions-only** Hugging Face dataset
+[`suchirsalhan/celex-en-phoneme-sample`](https://huggingface.co/datasets/suchirsalhan/celex-en-phoneme-sample)
+— access is granted to holders of a valid CELEX/LDC license
+([LDC96L14](https://catalog.ldc.upenn.edu/docs/LDC96L14/celex.readme.html)):
+
+```python
+from phoneme_entropy.data import load_hf_dataset
+ds = load_hf_dataset(split="train")   # after your access request is approved
+```
+
+## API
+
+| Import | Purpose |
+| --- | --- |
+| `Entropy`, `FreqShrink`, `JS_KullbackLeibler`, `JS_JensenShannon` | Entropy / divergence estimators (MLE, JSE, Chao-Shen, CWJ, NBRS, NSB), vendored. |
+| `estimate_alpha_entropy`, `dirichlet_entropy`, `plot_ranks` | Symmetric-Dirichlet inference and rank/probability curves. |
+| `segment_informativity`, `phoneme_prefix_entropy` | Segmental informativity over phoneme strings. |
+| `compute_maxent_from_matrix` | Maximum-entropy fit to feature expectations. |
+| `phoneme_entropy.data.load_sample`, `load_hf_dataset` | Data loaders. |
+
+## License
+
+MIT — see [`LICENSE`](LICENSE). Bundles third-party code; see
+[`THIRD_PARTY_LICENSES`](THIRD_PARTY_LICENSES). Please cite via
+[`CITATION.cff`](CITATION.cff).

phoneme_entropy-0.1.0/THIRD_PARTY_LICENSES

@@ -0,0 +1,37 @@
+This distribution bundles third-party code. Their licenses are reproduced below.
+
+===============================================================================
+entropy-estimators — https://github.com/fermosc24/entropy-estimators
+Vendored into: src/phoneme_entropy/estimators.py
+===============================================================================
+
+MIT License
+
+Copyright (c) 2025 fermosc24
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-------------------------------------------------------------------------------
+The NSB estimator portion of the vendored file is itself derived from:
+  * nsb_entropy.py by Sungho Hong (OIST, 2011), updated to Python 3 by
+    Charlie Strauss (LANL, 2019), based on Mathematica functions by
+    Christian Mendl. See http://nsb-entropy.sourceforge.net and
+    Nemenman, Shafee & Bialek, "Entropy and Inference, Revisited",
+    arXiv:physics/0108025.
+-------------------------------------------------------------------------------

phoneme_entropy-0.1.0/examples/quickstart.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""End-to-end quickstart for phoneme-entropy.
+
+Reproduces the workflow from the original tutorial notebook using the packaged
+API and the bundled CELEX sample. Run it after installing the package:
+
+    pip install -e ".[dev]"
+    python examples/quickstart.py
+
+It prints intermediate results and (if matplotlib is available) draws the
+predicted-vs-observed rank plot and the max-ent fit.
+"""
+
+from collections import defaultdict
+
+import numpy as np
+import pandas as pd
+
+from phoneme_entropy import (
+    compute_maxent_from_matrix,
+    estimate_alpha_entropy,
+    phoneme_prefix_entropy,
+    segment_informativity,
+)
+from phoneme_entropy.data import load_sample
+
+
+def main() -> None:
+    # 1. Load the bundled CELEX English sample. "Word" holds space-separated
+    #    phonemes; "Frequency" holds corpus frequencies.
+    sample = load_sample()
+    print(f"Loaded {len(sample)} words. Columns: {list(sample.columns)}")
+
+    # 2. Phoneme unigram frequencies.
+    phonfreqs = defaultdict(int)
+    for word in sample["Word"]:
+        for phoneme in word.split():
+            phonfreqs[phoneme] += 1
+    phonfreqs = pd.DataFrame(phonfreqs.items(), columns=["Phoneme", "Frequency"])
+    print(f"Inventory size (V): {len(phonfreqs)}")
+
+    # 3. Symmetric-Dirichlet concentration implied by the phoneme distribution.
+    #    n_boot=0 -> deterministic point estimate (no bootstrap standard error).
+    alpha, se = estimate_alpha_entropy(phonfreqs["Frequency"], n_boot=0)
+    print(f"Estimated Dirichlet alpha: {alpha:.4f}")
+
+    # 4. Per-phoneme surprisal (next-phoneme informativity) and prefix entropy.
+    surprisal = segment_informativity(sample["Word"])
+    prefix_entropy = phoneme_prefix_entropy(sample["Word"])
+
+    # 5. Merge, form frequency-weighted target expectations, fit max-ent.
+    merged = phonfreqs.merge(surprisal, on="Phoneme").merge(prefix_entropy, on="Phoneme")
+    p = merged["Frequency"] / merged["Frequency"].sum()
+    targets = np.array(
+        [np.sum(p * merged["Surprisal"]), np.sum(p * merged["Entropy"])]
+    )
+    F = merged[["Surprisal", "Entropy"]].to_numpy()
+    probs, lambdas = compute_maxent_from_matrix(F, targets)
+    print(f"Max-ent Lagrange multipliers (lambda): {lambdas}")
+
+    # 6. Optional plots (only if matplotlib is installed).
+    try:
+        import matplotlib.pyplot as plt
+
+        from phoneme_entropy.dirichlet import plot_ranks
+
+        plot_ranks(phonfreqs["Frequency"], types="Reversed", loglog=False)
+        plt.figure()
+        plt.scatter(np.log(p), np.log(probs))
+        plt.xlabel("(log) observed probability")
+        plt.ylabel("(log) predicted probability")
+        plt.tight_layout()
+        plt.show()
+    except ImportError:
+        print("matplotlib not installed; skipping plots. Install with '.[plot]'.")
+
+
+if __name__ == "__main__":
+    main()