trimbed 0.0.1__tar.gz

trimbed-0.0.1/.gitattributes

@@ -0,0 +1,12 @@
+# Line-ending policy: LF in the repository, LF in every working tree.
+#
+# `text=auto` lets Git detect which files are text (binary content is left untouched),
+# and `eol=lf` pins the checkout ending on every platform rather than following
+# core.autocrlf. Without this, files committed from Windows keep their CRLF, which shows
+# up as whole-file diffs and breaks tools that parse the repo's own config from a shell
+# (hence the `tr -d '\r'` guard in the Makefile's `test-matrix` target).
+* text=auto eol=lf
+
+# Generated by uv and only ever rewritten wholesale; collapse it in diffs and keep it out
+# of review noise.
+uv.lock linguist-generated=true

trimbed-0.0.1/.github/workflows/ci.yml

@@ -0,0 +1,114 @@
+name: CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [main]
+    paths:
+      - "**.py"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".pre-commit-config.yaml"
+      - ".github/workflows/ci.yml"
+      - "Makefile"
+      - "mkdocs.yml"
+      - "docs/**"
+  pull_request:
+    branches: [main]
+    paths:
+      - "**.py"
+      - "pyproject.toml"
+      - "uv.lock"
+      - ".pre-commit-config.yaml"
+      - ".github/workflows/ci.yml"
+      - "Makefile"
+      - "mkdocs.yml"
+      - "docs/**"
+
+# Cancel any in-progress run for the same branch/PR when a new push arrives.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quality:
+    name: Code quality
+    runs-on: ubuntu-latest
+    steps:
+      # fetch-depth: 0 everywhere because hatch-vcs derives the version from the tags.
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: uv sync --locked --group dev
+
+      # The hooks already cover ruff lint, ruff format and `uv lock --check`, so there is
+      # no separate lint step here. There is deliberately no type-checking step: type
+      # hints in this project are documentation, not a contract.
+      - name: Run pre-commit
+        run: uv run pre-commit run --all-files --show-diff-on-failure
+
+  # Validation only: builds the site into a throwaway directory and discards it. Nothing
+  # is published here -- docs.yml is the only workflow that touches gh-pages, and it only
+  # runs on a release. The point is to catch broken cross-references, dead anchors and
+  # missing nav entries on the pull request that introduces them rather than at release
+  # time.
+  docs:
+    name: Docs build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Install docs dependencies
+        run: uv sync --locked --group docs
+
+      - name: Build docs (strict)
+        run: uv run mkdocs build --strict --site-dir "${RUNNER_TEMP}/site"
+
+  test:
+    name: Tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # Not 3.14: `datasets` pulls in `multiprocess`, which ships no cp314 wheels and
+        # fails to import there, and `datasets` is a hard runtime dependency.
+        python-version: ["3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --locked --group dev
+
+      # The offline suite: no Hub access, no real weights. See hub-tests.yml for those.
+      - name: Run tests
+        run: make test
+
+      - name: Upload coverage
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          fail_ci_if_error: true
+          disable_search: true
+          disable_telem: true

trimbed-0.0.1/.github/workflows/docs.yml

@@ -0,0 +1,93 @@
+name: Publish docs to GitHub Pages
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: Git tag to publish docs for, for example v0.3.0
+        required: true
+        type: string
+
+# Cancel any in-progress docs deploy for the same branch when a new push arrives.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Resolve docs tag and version
+        id: docs_ref
+        run: |
+          if [ "${{ github.event_name }}" = "release" ]; then
+            DOCS_GIT_TAG="${{ github.event.release.tag_name }}"
+          else
+            DOCS_GIT_TAG="${{ inputs.release_tag }}"
+          fi
+
+          if [ -z "${DOCS_GIT_TAG}" ]; then
+            echo "A release tag is required to publish docs." >&2
+            exit 1
+          fi
+
+          if ! git rev-parse --verify "refs/tags/${DOCS_GIT_TAG}" >/dev/null 2>&1; then
+            echo "Git tag '${DOCS_GIT_TAG}' does not exist." >&2
+            exit 1
+          fi
+
+          DOCS_VERSION="${DOCS_GIT_TAG#v}"
+          if [ -z "${DOCS_VERSION}" ] || [ "${DOCS_VERSION}" = "${DOCS_GIT_TAG}" ]; then
+            echo "Release tag must start with 'v'; got '${DOCS_GIT_TAG}'." >&2
+            exit 1
+          fi
+
+          echo "docs_git_tag=${DOCS_GIT_TAG}" >> "$GITHUB_OUTPUT"
+          echo "docs_version=${DOCS_VERSION}" >> "$GITHUB_OUTPUT"
+          # docs/hooks.py reads this to point every "View source on GitHub" link at the
+          # tag these docs were built from instead of at main.
+          echo "DOCS_SOURCE_REF=${DOCS_GIT_TAG}" >> "$GITHUB_ENV"
+
+      - name: Check out docs tag
+        run: git checkout "${{ steps.docs_ref.outputs.docs_git_tag }}"
+
+      # Docs *content* comes from the tag; the tooling that renders it always comes from
+      # main, so a rendering fix reaches published docs without re-tagging. Pinned to main
+      # rather than GITHUB_SHA because GITHUB_SHA is the tag commit on a release but the
+      # branch tip on a workflow_dispatch, which would make a manual rerun build a
+      # different site than the release did.
+      - name: Use current docs build tooling from main
+        run: |
+          git checkout origin/main -- \
+            pyproject.toml uv.lock mkdocs.yml docs/hooks.py docs/overrides
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Install docs dependencies
+        run: uv sync --locked --group docs
+
+      - name: Configure git identity
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Fetch gh-pages branch for mike
+        run: git fetch origin gh-pages --depth=1 || true
+
+      - name: Deploy release docs (tag -> version + set as latest)
+        run: |
+          VERSION="${{ steps.docs_ref.outputs.docs_version }}"
+          uv run mike deploy --push --update-aliases "${VERSION}" latest
+          uv run mike set-default --push latest

trimbed-0.0.1/.github/workflows/hub-tests.yml

@@ -0,0 +1,57 @@
+# The tests that reach the Hugging Face Hub, kept off the pull-request path on purpose: a
+# Hub outage or a several-gigabyte download must never be what blocks a merge. Run them
+# on a schedule and on demand instead.
+name: Hub tests
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Mondays, 05:40 UTC
+    - cron: "40 5 * * 1"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tokenizers:
+    name: Hub tokenizer trims
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      # --all-extras: the mT5 case needs trimbed[convert] to read a bare spiece.model.
+      - name: Install dependencies
+        run: uv sync --locked --all-extras --group dev
+
+      - name: Run Hub tokenizer tests
+        run: make test-network
+
+  models:
+    name: Hub model trims
+    runs-on: ubuntu-latest
+    # Minutes per case, and every one of them downloads a real checkpoint. A failure here
+    # is worth seeing but is not a reason to call the run red.
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: uv sync --locked --all-extras --group dev
+
+      - name: Run Hub model tests
+        run: make test-slow

trimbed-0.0.1/.github/workflows/publish.yml

@@ -0,0 +1,35 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi-publish:
+    name: Build and Publish
+    runs-on: ubuntu-latest
+
+    environment: release
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      # fetch-depth: 0 so hatch-vcs sees the tag and builds a released version rather
+      # than a development one.
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+
+      - name: Build package
+        run: uv build
+
+      # Trusted publishing: the OIDC token above stands in for an API token.
+      - name: Publish to PyPI
+        run: uv publish

trimbed-0.0.1/.gitignore

@@ -0,0 +1,77 @@
+.ruff_cache/
+papers/
+data/
+configs/
+
+# Python caches and bytecode
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environments
+.venv/
+venv/
+env/
+# without slash because we are using a sym-link as the venv
+# lives on scratch
+.venv
+
+# Build / packaging artifacts
+build/
+dist/
+*.egg-info/
+
+# Test / coverage artifacts
+.pytest_cache/
+.coverage
+.coverage.*
+coverage.xml
+report.xml
+htmlcov/
+
+# uv's cache, relocated into the project directory by CI
+.uv-cache/
+
+# One venv per interpreter, written by `make test-matrix`
+.venvs/
+
+# mkdocs output; the published site lives on the gh-pages branch
+/site
+
+# Jupyter checkpoints
+.ipynb_checkpoints/
+
+# Training outputs and checkpoints
+runs/
+checkpoints/
+
+# Model and large binary artifacts
+*.pt
+*.bin
+*.safetensors
+
+# System/editor files
+.DS_Store
+.vscode/
+.idea/
+
+# Logs
+logs/
+*.log
+
+# Downloaded / derived data
+data/dev_samples/
+
+# Local planning notes
+plan.md
+
+# Claude Code local configuration
+.claude/
+CLAUDE.md
+.agents/
+
+# upstream bug reports
+bugs/
+
+# Trimmed artefacts (the default output_dir in README/examples)
+trimmed/

trimbed-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,76 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-toml
+      - id: check-yaml
+      # Backs up .gitattributes: that pins what lands in the index, this catches CRLF in
+      # the working tree before it gets there.
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: local
+    hooks:
+      # mkdocstrings renders docstrings as Markdown, so a Sphinx role survives as literal
+      # ":class:" text on the site. Use [`Name`][full.dotted.path].
+      - id: no-rst-roles
+        name: no Sphinx/RST roles in docstrings
+        language: pygrep
+        entry: ':(class|meth|func|mod|attr|obj|exc|data|ref):`'
+        types: [python]
+        files: ^src/trimbed/
+      # Griffe only documents an attribute that carries a real docstring, so a "#:"
+      # comment renders as nothing at all. Put a """docstring""" under the assignment.
+      - id: no-sphinx-attribute-comments
+        name: no Sphinx "#:" attribute comments
+        language: pygrep
+        entry: '^\s*#:'
+        types: [python]
+        files: ^src/trimbed/
+      # Only the staged files are passed in, restricted to our own source dirs by
+      # `files` below. Linting whole directories instead makes pre-commit's
+      # stash-and-restore of unstaged edits conflict with the hooks' own auto-fixes.
+      - id: ruff-lint
+        name: ruff lint
+        entry: uv run --frozen ruff check --fix --exit-non-zero-on-fix --force-exclude
+        language: system
+        types_or: [python, pyi]
+        files: ^(src/trimbed|tests|examples|docs)/
+        # ruff parallelises internally, so hand it the whole file list in one process
+        # rather than letting pre-commit fan it out across several.
+        require_serial: true
+      - id: ruff-format
+        name: ruff format
+        entry: uv run --frozen ruff format --force-exclude
+        language: system
+        # ruff formats the Python blocks inside Markdown too, which is what keeps the
+        # examples in docs/ from drifting out of the project's own style.
+        types_or: [python, pyi, markdown]
+        files: ^(src/trimbed|tests|examples|docs)/
+        require_serial: true
+      # `uv` is already on PATH and every other hook runs under `uv run --frozen`, so a
+      # pyproject.toml edit without a matching re-lock breaks all of them at once.
+      - id: uv-lock-check
+        name: uv lock check
+        entry: uv lock --check
+        language: system
+        files: ^(pyproject\.toml|uv\.lock)$
+        pass_filenames: false
+
+      # Pre-push only, and only when Python actually changed: runs the offline suite
+      # against every interpreter in the CI matrix, so version-specific breakage is caught
+      # here instead of by a red job after the push. Requires
+      # `pre-commit install --hook-type pre-push` once per clone; `pre-commit run
+      # --all-files` (what CI runs) skips it, since that defaults to the pre-commit stage.
+      - id: test-matrix
+        name: pytest on every CI Python version
+        entry: make test-matrix
+        language: system
+        stages: [pre-push]
+        types: [python]
+        pass_filenames: false

trimbed-0.0.1/.python-version

@@ -0,0 +1 @@
+3.13

trimbed-0.0.1/CONTRIBUTING.md

@@ -0,0 +1,103 @@
+# Contributing to trimbed
+
+## Setup
+
+The project is managed with [uv](https://docs.astral.sh/uv/). One command gets you a
+working environment with every development tool:
+
+```bash
+uv sync --locked --group dev
+uv run pre-commit install
+uv run pre-commit install --hook-type pre-push
+```
+
+The second `pre-commit install` is not optional if you want the matrix check below to run.
+It is a separate hook stage, so it needs its own registration once per clone.
+
+Python 3.12 or 3.13. Not 3.14 yet: `datasets` pulls in `multiprocess`, which has no cp314
+wheels and fails to import there, so `requires-python` caps at `<3.14`.
+
+Runtime dependencies are capped at the next minor rather than the next major, because
+trimbed works against internals of `transformers`, `tokenizers`, `skeletoken` and `torch`
+that move on minor releases. Raising a cap is its own change, with `make test-all` behind
+it, not something to fold into an unrelated pull request.
+
+## The checks
+
+```bash
+make style         # ruff check --fix + ruff format
+make quality       # the non-mutating equivalent
+make test          # the offline suite, with coverage
+make test-network  # + the Hub tokenizer trims
+make test-slow     # + the Hub model trims; minutes per case
+make test-all      # everything
+make build-docs    # mkdocs build --strict
+make serve-docs    # live preview on localhost
+```
+
+Run `make style`, `make test` and `make build-docs` before finishing any change.
+
+The suite is offline by default and covers **100% of statements and branches**. Keep it
+there. The gate is Codecov rather than a number in `pyproject.toml`: `codecov.yml` fails a
+pull request whose project coverage drops more than 3% against the base branch, or whose
+own changed lines are less than 90% covered. A handful of genuinely unreachable guards
+carry `# pragma: no cover` with a comment saying which invariant makes them unreachable.
+Prefer a real test over a new pragma.
+
+`make test-matrix` runs the offline suite once per interpreter in the CI matrix. It reads
+the version list out of `.github/workflows/ci.yml`, so the two cannot drift. It is also the
+pre-push hook, which is why version-specific breakage is caught before it becomes a red job.
+
+## Style
+
+- Python 3.12+: `|` unions, `type` aliases, PEP 695 generics, `Self`.
+- Concise Google-style docstrings on every public function, enforced by ruff's `D` rules.
+- Type hints are documentation, not a contract. There is no mypy, no ruff `ANN` rules and
+  no `py.typed`, and none of the three is wanted.
+- Raise built-in exceptions. `MissingDependencyError` is the only custom one, and it
+  subclasses `ImportError`.
+- No `# noqa` in the source. A rule worth suppressing in more than one place is ignored in
+  `pyproject.toml`, with the reason next to it.
+- Comments explain *why*, not what.
+
+## Documentation
+
+The site is MkDocs Material with mkdocstrings. Guides live in `docs/*.md`; the API
+reference is one stub per module under `docs/api/`, each holding a title and a single
+`::: trimbed.<module>` directive, and each listed in `mkdocs.yml`'s `nav`.
+
+Two rules keep it navigable:
+
+- **Cross-reference trimbed's own symbols** as ``[`Name`][trimbed.module.Name]``. Sphinx
+  roles (`` :class:`Name` ``) render as literal text on the site and are blocked by a
+  pre-commit hook. External names (`save_pretrained`, `tokenizers.Tokenizer`) stay as
+  plain backticks.
+- **Document an attribute with a docstring**, not a `#:` comment. Griffe only picks up the
+  former; the latter renders as nothing at all. Also blocked by a pre-commit hook.
+
+Adding a module means adding its stub page and its nav entry in the same pass.
+`tests/test_docs.py` fails if either is missing, and `mkdocs build --strict` fails on any
+cross-reference that does not resolve.
+
+`examples/` and `tests/` must be kept in sync with `src/` the same way.
+`tests/test_examples.py` executes every example against tiny in-process fixtures, so a
+stale example fails the suite rather than rotting quietly.
+
+## Pull requests
+
+CI runs on every pull request to `main`: pre-commit over the whole tree, a strict docs
+build, and the offline suite on Python 3.12 and 3.13 with coverage uploaded to Codecov.
+The Hub tests are a separate scheduled workflow, deliberately off the pull-request path so
+that a Hub outage or a several-gigabyte download never blocks a merge. You can run them on
+demand from the Actions tab.
+
+## Releases
+
+Versions come from git tags through hatch-vcs; nothing is hand-edited.
+
+1. Tag the commit `vX.Y.Z` and push the tag.
+2. Publish a GitHub Release for it.
+
+That fires two workflows in parallel: `publish.yml` builds and uploads to PyPI through
+trusted publishing, and `docs.yml` deploys the docs for that version with mike and moves
+the `latest` alias. The tag must start with `v`, which the docs workflow checks.