batchalign 0.10.0__cp310-abi3-win_amd64.whl

batchalign/BUILD.bazel

@@ -0,0 +1,376 @@
+"""Bazel-native dev / test path for the batchalign Python package.
+
+Runtime targets (no maturin, no uv at `bazel run` time):
+
+    bazel run  //python/batchalign            # CLI (py_binary)
+    bazel test //python/batchalign:tests      # pytest (py_test)
+    bazel build //python/batchalign           # py_binary + .so + py_library
+
+The cdylib is built natively by `rust_shared_library` in
+`//crates/batchalign/batchalign-engine` and lands at `_core.so`
+alongside the Python sources in the runfiles tree.
+
+Release-only (still maturin — sole reader of pyproject.toml's full
+multi-platform tagging matrix + optional-deps groups):
+
+    bazel run //python/batchalign:wheel       # build + install under
+                                              # python/target/wheels/<tag>.whl
+
+The wheel command is one target end-to-end: it triggers the maturin
+build (cached as `:_wheel_artifact`, also depended on by `:sidecar`)
+and copies the result into `python/target/wheels/<tag>.whl` with the
+PyPI-tagged basename. No multi-step incantations; no separate `bazel
+build` + `cp` recipe for contributors or CI to memorize.
+
+PyPI uploads are CI-only via trusted publishing
+(`.github/workflows/publish-pypi.yml`). There is no local-publish
+target — local twine uploads invite the wrong mental model and bypass
+the OIDC + branch-protection guarantees that the CI flow provides.
+
+`pyproject.toml` is the single source of truth for the wheel
+(version, runtime deps, all `[project.optional-dependencies]`
+groups, scripts, classifiers).
+"""
+
+# Zero hand-listed deps. `all_runtime_deps()` returns every package in
+# requirements.lock.txt, with rules_python's marker-conditional dangling
+# names filtered out (see `//bazel/python:deps.bzl` for the why; the
+# filter is a per-name list of *marker-excluded names from
+# requirements.lock.txt*, not a list of deps to include).
+load("//bazel/python:deps.bzl", "all_runtime_deps")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+load("@rules_shell//shell:sh_binary.bzl", "sh_binary")
+
+package(default_visibility = ["//visibility:public"])
+
+_RUNTIME_DEPS = all_runtime_deps()
+
+py_library(
+    name = "batchalign_lib",
+    # `_core/` is a Bazel subpackage (it owns the proto-codegen genrule),
+    # so glob stops there. We pull the `_core` sources back in by
+    # depending on its `:proto` py_library, which exports
+    # `batchalign._core.{__init__,proto,_proto_generated}` plus the
+    # staged `_core.so`.
+    srcs = glob(
+        ["**/*.py"],
+        exclude = ["tests/**"],
+    ),
+    # Non-Python data shipped with backends (e.g. BA2's disfluency /
+    # replacement word-lists `backends/utseg/support/*.eng` that the
+    # CHATUtterance segmenter reads at runtime). Without these in runfiles
+    # the lists load empty and `uh` never becomes `&-uh`.
+    data = glob(["**/support/*"]),
+    # `imports = [".."]` puts python/ on PYTHONPATH so `import batchalign`
+    # resolves. The package layout is python/batchalign/{__init__,...}.py.
+    imports = [".."],
+    deps = _RUNTIME_DEPS + ["//python/batchalign/_core:proto"],
+)
+
+# Default runnable target — `bazel run //python/batchalign` invokes
+# `python -m batchalign` against the Bazel-built `.so`.
+py_binary(
+    name = "batchalign",
+    srcs = ["__main__.py"],
+    main = "__main__.py",
+    deps = [":batchalign_lib"],
+)
+
+py_test(
+    name = "pytest",
+    srcs = glob(["tests/**/*.py"]) + ["_pytest_runner.py"],
+    main = "_pytest_runner.py",
+    # The tests form a package (`tests/__init__.py`). Pytest's default
+    # `prepend` mode resolves their source symlinks and inserts the checkout's
+    # `python/` directory ahead of Bazel's runfiles package. That is invisible
+    # on developer machines with a maturin-built `_core.abi3.so`, but clean CI
+    # then loses the staged `_core.so`. Importlib mode leaves Bazel's package
+    # root authoritative throughout collection.
+    args = ["python/batchalign/tests", "-q", "--import-mode=importlib"],
+    # Vendored parity fixtures used by `test_golden_fixtures.py`. Must
+    # be in runfiles for the test to find them — without this they were
+    # silently skipped on every CI run (the earlier copy lived in a
+    # developer-local `talkbank-alignment/` directory). Same for the
+    # Rust NUM2LANG table + the E316 spec, both probed by the hermetic
+    # Landing 3 #16/#17 guards.
+    data = [
+        "//resources/test_fixtures:parity_fixtures",
+        "//crates/batchalign/batchalign-core:asr_data",
+        "//resources/spec/errors:E316_angle_bracket_in_mor_stem.md",
+    ],
+    # all_requirements already contains pytest + pytest-xdist via the
+    # dev extra in pyproject; don't re-list them or Bazel errors on
+    # duplicate-deps.
+    deps = [":batchalign_lib"] + _RUNTIME_DEPS,
+)
+
+# ---------------------------------------------------------------------------
+# Maturin-driven wheel (release path).
+# ---------------------------------------------------------------------------
+# Two-layer shape:
+#
+#   :_wheel_artifact      genrule. Produces `batchalign.whl` (wheel
+#                          bytes) + `wheel_tag.txt` (maturin's PyPI-
+#                          tagged basename, e.g.
+#                          `batchalign-0.7.1-cp312-cp312-macosx_11_0_arm64.whl`).
+#                          Tagged `no-cache` because maturin reads
+#                          `.rs` / `.py` / `pyproject.toml` from the
+#                          source tree directly, not through Bazel
+#                          inputs — letting Bazel always re-invoke
+#                          maturin (cargo's incremental cache makes the
+#                          no-op case sub-second) is cheaper than
+#                          declaring every Rust crate target in `srcs`
+#                          for invalidation and paying for a parallel
+#                          rules_rust build of every crate maturin
+#                          already compiles. Not user-facing — both
+#                          `:sidecar` and `:wheel` consume it.
+#
+#   :wheel                 sh_binary. `bazel run //python/batchalign:wheel`
+#                          builds :_wheel_artifact, reads the tag, and
+#                          copies the wheel into
+#                          `python/target/wheels/<tag>.whl`. This is
+#                          the single user-facing wheel command —
+#                          contributors, the justfile, and CI all
+#                          invoke it the same way. No
+#                          memorized-shell-snippet alternative.
+#
+# Hermetic C toolchain. The toolchains_llvm-managed clang + ar +
+# ranlib are passed as execpaths; the shell scripts resolve them and
+# export CC/CXX/AR/RANLIB before invoking maturin. cc-rs inside
+# cargo then picks up the pinned compiler instead of /usr/bin/cc,
+# which decouples every `*-sys` crate build from the host
+# CommandLineTools state. See MODULE.bazel's `toolchains_llvm` block
+# for the cflag injection that papers over the macOS 26.x SDK
+# `__kernel_ptr_semantics` regression.
+#
+# `$(COMPILATION_MODE)` is a Bazel make-variable that expands to
+# `opt`, `dbg`, or `fastbuild` based on `-c <mode>`. The shell scripts
+# translate that into maturin's `--release` / dev profile, so users
+# never need PYAPP_PROFILE / MATURIN_PROFILE env vars — `bazel run
+# -c opt //...` is the only knob.
+#
+# Historical note: this used to be two parallel targets (a
+# `sh_binary :wheel` that ran maturin and dropped a wheel into the
+# source tree, plus a `:wheel_file` genrule that ran maturin a second
+# time for Bazel-tracked output). Editing a single .rs file forced
+# two maturin runs. Collapsed: one genrule produces the wheel bytes;
+# the sh_binary is now a thin copy-out that reuses that output. A
+# follow-up pass also stripped the rules_rust crate targets from
+# `srcs` — they were never consumed by maturin, only triggering a
+# Bazel-side rust_library build that doubled the rustc cost on every
+# `.rs` edit. The `no-cache` tag below preserves correctness without
+# that cost.
+#
+# Caveat — the rules_rust build isn't fully eliminated. `srcs` pulls
+# in `:_proto_generated_py`, which transitively depends on
+# `//crates/batchalign/batchalign-core:proto_schema_json`. That genrule
+# uses `:emit_proto_schema` (a rust_binary) as its tool, and
+# `:emit_proto_schema` links `:batchalign_core` as a rust_library.
+# Result: every wheel build first compiles `batchalign_core` + a chunk
+# of its transitive crate-universe deps via rules_rust in the exec
+# config (visible as `Compiling Rust rlib batchalign_core ... [for
+# tool]`), then maturin rebuilds the same crate graph from scratch
+# under cargo's target/. Two compiles of overlapping crates per cold
+# wheel build. We accept this for now — cargo's incremental cache
+# absorbs the hit on warm rebuilds, and the schema-emitter-as-tool
+# pattern is what gives us regen-on-build for `_proto_generated.py`
+# without a hand-checked-in artifact. If this cost ever bites, three
+# escape hatches: (a) check in `proto.schema.json` with a CI diff
+# guard, (b) factor proto types into a leaf `batchalign-proto` crate
+# so `:emit_proto_schema` doesn't pull all of `batchalign_core`, or
+# (c) move schema emission into maturin's build itself and ship the
+# generated `.py` inside the wheel.
+config_setting(
+    name = "windows",
+    constraint_values = ["@platforms//os:windows"],
+)
+
+_LLVM_TOOLS = select({
+    ":windows": [],
+    "//conditions:default": [
+        "//bazel/python:llvm_clang",
+        "//bazel/python:llvm_ar",
+        "//bazel/python:llvm_ranlib",
+    ],
+})
+
+genrule(
+    name = "_wheel_artifact",
+    outs = ["batchalign.whl", "wheel_tag.txt"],
+    srcs = [
+        # `_proto_generated.py` is the only input the genrule action
+        # actually *reads via Bazel runfiles* — maturin_build.sh stages
+        # it into the source tree before maturin packages. The `.rs`
+        # sources under `crates/` and `.py` sources under
+        # `python/batchalign/` are read directly via
+        # `BUILD_WORKSPACE_DIRECTORY`, NOT through Bazel inputs (see the
+        # `tags` comment below for why).
+        "//python/batchalign/_core:_proto_generated_py",
+        # `pyproject.toml` IS declared as a Bazel input. It owns the
+        # wheel version (`[project].version`), the runtime deps, the
+        # optional-dependencies extras, and the maturin config — every
+        # one of these changes the wheel's identity or contents. Without
+        # it in `srcs`, a version bump in pyproject.toml does not
+        # invalidate this action and `bazel run //python/batchalign:wheel`
+        # silently serves the previous wheel. The `tags = ["no-cache"]`
+        # below does NOT compensate for this: that tag disables result
+        # caching, not action skipping — Bazel's incremental analysis
+        # still skips when declared inputs are unchanged. Declaring a
+        # plain TOML file in `srcs` does not trigger any `rust_library`
+        # rebuild (that concern only applies to `.rs` files reached via
+        # rules_rust).
+        "//python:pyproject.toml",
+    ] + _LLVM_TOOLS,
+    tools = [
+        "//bazel/python:maturin_build.sh",
+        "//bazel/python:maturin_genrule.sh",
+        "//bazel/python:hermeticity_guard.sh",
+        "//bazel/python:runfiles_resolve.sh",
+        "@multitool//tools/uv",
+        "@rules_rust//rust/toolchain:current_cargo_files",
+        "@rules_rust//rust/toolchain:current_rustc_files",
+    ] + _LLVM_TOOLS,
+    cmd = " ".join([
+        "PYAPP_RUST_BIN_DIRS='$(execpaths @rules_rust//rust/toolchain:current_cargo_files) $(execpaths @rules_rust//rust/toolchain:current_rustc_files)'",
+        "$(execpath //bazel/python:maturin_genrule.sh)",
+        "$(location batchalign.whl)",
+        "$(location wheel_tag.txt)",
+        "$(execpath @multitool//tools/uv)",
+        "$(execpath //python/batchalign/_core:_proto_generated_py)",
+        "$(COMPILATION_MODE)",
+    ]) + select({
+        ":windows": " host",
+        "//conditions:default": " llvm " + " ".join([
+            "$(execpath //bazel/python:llvm_clang)",
+            "$(execpath //bazel/python:llvm_ar)",
+            "$(execpath //bazel/python:llvm_ranlib)",
+        ]),
+    }),
+    # `local = True`: maturin needs network access (cargo fetches) and a
+    # persistent target/ cache that a sandboxed action can't supply.
+    # uv, rustc, and cargo remain Bazel-managed. The C toolchain follows the
+    # platform contract in maturin_build.sh (LLVM, Xcode, or Visual Studio).
+    local = True,
+    # `no-cache`: disables caching of the action's outputs (local disk
+    # cache + remote cache). The `.rs` under `crates/` and `.py` under
+    # `python/batchalign/` are read from the source tree at run time,
+    # NOT through Bazel's declared input set — if we declared all of
+    # them in `srcs`, every `.rs` edit would force a parallel Bazel-side
+    # `rust_library` rebuild of every crate maturin already compiles
+    # via cargo (rustc twice per edit). Skipping the declaration keeps
+    # the GUI sidecar dev loop fast; cargo's incremental cache and
+    # maturin's "already up to date" check keep the no-op case
+    # sub-second.
+    #
+    # Caveat: `no-cache` does NOT force re-execution. Bazel's
+    # incremental analysis still skips this action when its declared
+    # input hashes are unchanged. That is why `pyproject.toml` is an
+    # explicit `srcs` entry — without it, a version bump silently
+    # serves the previously built wheel. Source-tree-resident `.rs` /
+    # `.py` edits also do not invalidate this action on their own;
+    # they're picked up when some declared input (e.g. the proto
+    # generator) changes or when the user forces a rebuild
+    # (`bazel clean`, deleting `bazel-bin/python/batchalign/*.whl`, or
+    # touching pyproject.toml). If pure `.rs`/`.py` invalidation
+    # becomes a real pain point, the next step is to declare those
+    # globs in `srcs` and accept the double-build cost, or move the
+    # whole wheel build out of a genrule into a `repository_rule` /
+    # external action with explicit input tracking.
+    tags = ["no-cache"],
+)
+
+# User-facing wheel command. `bazel run //python/batchalign:wheel`
+# (and `just batchalign wheel`, and CI wheel jobs) all funnel through
+# this one target. It reads the cached `:_wheel_artifact` outputs from
+# runfiles and copies them into `python/target/wheels/<tag>.whl`
+# inside the source tree. No second maturin run; no shell snippet
+# contributors need to memorize.
+sh_binary(
+    name = "wheel",
+    srcs = ["//bazel/python:install_wheel.sh"],
+    args = [
+        "$(rootpath :batchalign.whl)",
+        "$(rootpath :wheel_tag.txt)",
+    ],
+    data = [
+        ":batchalign.whl",
+        ":wheel_tag.txt",
+    ],
+)
+
+# Standalone production-daemon sidecar binary built via PyApp.
+#
+# `bazel build //python/batchalign:sidecar` produces a self-contained
+# executable at `bazel-bin/python/batchalign/sidecar` that bundles
+# CPython + the batchalign wheel + the `[api]` extra. The binary IS
+# the daemon — running it boots `batchalign3 daemon` directly without
+# the user typing the subcommand. Named "sidecar" because that's the
+# role it plays in a deployment (companion service next to the main
+# CLI / wheel install, not a separate top-level app).
+#
+# Architecture (all Bazel-managed, no host tool requirements beyond
+# bazel + just):
+#   - Wheel input from `:batchalign.whl` (one of `:_wheel_artifact`'s
+#     outputs). Depending on the specific output rather than the
+#     genrule label keeps `wheel_tag.txt` out of the sidecar's input
+#     set (clean and explicit; not strictly required since the tag
+#     file is small and irrelevant to PyApp).
+#   - pyapp source from `@pyapp_src` (Bazel `git_repository`, pinned
+#     by SHA in MODULE.bazel; fetched lazily on first build).
+#   - cargo + rustc from `@rules_rust//rust/toolchain` (PATH-injected
+#     by `pyapp_install.sh` so the inner `cargo install` never sees
+#     host cargo).
+#
+# `local = True` is required: pyapp's build.rs writes into its source
+# dir at compile time (an upstream design choice — see
+# bazel/python/pyapp_install.sh for the writable-copy workaround).
+# All other hermeticity holds.
+genrule(
+    name = "sidecar",
+    outs = ["sidecar"],
+    srcs = [
+        ":batchalign.whl",
+        # pyapp source is fetched lazily via git_repository in
+        # MODULE.bazel — Bazel only clones when this target is
+        # actually built.
+        "@pyapp_src//:srcs",
+        "@pyapp_src//:Cargo.toml",
+    ],
+    tools = [
+        "//bazel/python:pyapp_install.sh",
+        # Bazel-provided cargo + rustc. PATH-injected by the script
+        # via PYAPP_RUST_BIN_DIRS, so the inner `cargo install`
+        # never touches host cargo.
+        "@rules_rust//rust/toolchain:current_cargo_files",
+        "@rules_rust//rust/toolchain:current_rustc_files",
+    ],
+    # `local = True` is required: pyapp's build.rs writes into its
+    # source dir (an upstream design choice); cargo install's tmpdir
+    # copy of the source satisfies that, but only if cargo can use a
+    # persistent build cache — which sandboxed actions can't supply.
+    # All other hermeticity holds: cargo + rustc from rules_rust,
+    # pyapp source from @pyapp_src (Bazel-fetched, pinned SHA), wheel
+    # from :batchalign.whl. No crates.io network, no host cargo.
+    cmd = " ".join([
+        "PYAPP_RUST_BIN_DIRS='$(execpaths @rules_rust//rust/toolchain:current_cargo_files) $(execpaths @rules_rust//rust/toolchain:current_rustc_files)'",
+        "$(execpath //bazel/python:pyapp_install.sh)",
+        "$@",
+        "$(execpath //python/batchalign:batchalign.whl)",
+        "$(execpath @pyapp_src//:Cargo.toml)",
+        "$(COMPILATION_MODE)",
+    ]),
+    local = True,
+)
+
+# Lint kept as a shell-out because mypy/ruff configs sit in the uv-managed
+# venv. Could become a py_test rule later; for now sh_binary is enough.
+sh_binary(
+    name = "lint",
+    srcs = ["//bazel/python:lint.sh"],
+    args = ["$(rootpath @multitool//tools/uv)"],
+    data = [
+        "@multitool//tools/uv",
+        "//bazel/python:hermeticity_guard.sh",
+        "//python:pyproject.toml",
+    ],
+)

batchalign/README.md

@@ -0,0 +1,140 @@
+# batchalign
+
+TalkBank CHAT processing pipeline — ASR, forced alignment, morphosyntax
+(`%mor` / `%gra`), utterance segmentation, translation, and compare.
+The user-facing Python package; the runtime is a PyO3 extension backed
+by the Rust crates in `crates/batchalign/`.
+
+User and developer docs live in `book/src/batchalign/` (the mdBook is
+the source of truth). See `book/src/batchalign/developer/building.md`
+for the canonical build recipe.
+
+## Install
+
+From PyPI (stable wheels):
+
+```bash
+pip install batchalign
+```
+
+From source: **use the `just` recipes** — they go through Bazel, so
+every dep (proto codegen, Rust crates, PyO3 extension, Python wheel
+extras) is materialized for you. Don't reach for `maturin develop` or
+`uv sync` directly unless you're debugging the build itself.
+
+```bash
+just batchalign build              # build every Bazel target for batchalign
+just batchalign test               # run every Bazel test target
+just batchalign cli --help         # run `batchalign3` via the development bridge
+just batchalign pytest             # pytest (with full Bazel dep graph)
+just batchalign wheel              # host-platform wheel at python/target/wheels/
+just batchalign sidecar            # standalone daemon binary via PyApp
+just batchalign lint               # mypy (+ ruff)
+just batchalign versions           # source-of-truth version readout
+```
+
+`just --list batchalign` shows the full recipe list. The `just`
+recipes call into Bazel, so `tools/bazel` (the bundled wrapper) takes
+care of:
+
+- regenerating the pydantic-v2 wire types from
+  `crates/batchalign/batchalign-core/src/proto/*.rs`
+- rebuilding the `batchalign_core` PyO3 cdylib via maturin under the
+  hood
+- staging the binary into the wheel
+- propagating dependency changes to dependent targets
+
+If you genuinely need `bazel` directly (because a recipe you want
+isn't wrapped):
+
+```bash
+bazel build //...           # everything
+bazel test //...            # everything
+bazel run //book:html       # static book HTML
+bazel run //apps/batchalign/batchalign-gui:openapi   # GUI OpenAPI codegen
+```
+
+The base wheel ships only the lightweight runtime. Heavy ML backends
+are gated behind extras — install only what you use:
+
+```bash
+pip install 'batchalign[whisper]'      # Whisper ASR
+pip install 'batchalign[malayalam]'    # Malayalam Wav2Vec2 XLSR ASR
+pip install 'batchalign[stanza]'       # morphosyntax (%mor / %gra)
+pip install 'batchalign[pyannote]'     # optional local Pyannote backend API
+pip install 'batchalign[revai]'        # Rev.AI cloud ASR
+pip install 'batchalign[google]'       # Gemini cloud ASR + diarization
+pip install 'batchalign[cantonese]'    # Cantonese pipeline (FunASR, Tencent)
+pip install 'batchalign[qwen3]'        # Qwen3 ASR + forced aligner
+pip install 'batchalign[nllb]'         # NLLB translation
+pip install 'batchalign[api]'          # FastAPI daemon (`batchalign3 daemon`)
+pip install 'batchalign[all]'          # everything
+```
+
+## CLI
+
+```bash
+batchalign3 --help
+
+batchalign3 transcribe input_dir -o output_dir --lang eng
+batchalign3 transcribe input_dir -o output_dir --lang eng --engine whisper --diarize --diarize-engine pyannote-ai
+batchalign3 diarize    input_dir -o output_dir --engine pyannote-ai
+batchalign3 align     input_dir -o output_dir --engine wav2vec
+batchalign3 morphotag input_dir -o output_dir --language en
+batchalign3 utseg     input_dir -o output_dir
+batchalign3 translate input_dir -o output_dir --target eng
+batchalign3 compare   input_dir gold_dir   -o output_dir
+
+batchalign3 version                    # banner, version, git SHA
+batchalign3 cache {path,stats,clear}   # local result cache
+batchalign3 daemon                     # FastAPI server (needs [api])
+```
+
+When `-o` is omitted, results are written back in place. The CLI accepts
+either a single CHAT/media file or a folder (walked recursively).
+
+The diarization CLI defaults to pyannoteAI cloud. Select the optional local
+backend with `--engine pyannote` (or `--diarize-engine pyannote` on
+`transcribe`). Configure the cloud API key in the shared Batchalign config file
+(or set `BATCHALIGN_PYANNOTE_KEY`):
+
+```ini
+[diarize]
+engine.pyannote.key = YOUR_API_KEY
+```
+
+## Programmatic API
+
+```python
+import batchalign as ba
+
+# Build a backend chain.
+pipeline = ba.recipes.morphotag(
+    stanza_backend=ba.StanzaBackend(lang="en"),
+)
+
+# Or compose your own.
+asr = ba.WhisperBackend(language=ba.LanguageCode.from_iso("eng"))
+utseg = ba.CHATUtteranceBackend(model="talkbank/CHATUtterance-en")
+pipeline = ba.recipes.transcribe(asr_backend=asr, utseg_backend=utseg)
+
+# Run.
+inputs = [ba.media_from_path("session.wav")]
+outcomes = list(pipeline.run(inputs))
+for outcome in outcomes:
+    outcome.write("session.cha")
+```
+
+## Repository layout
+
+This package is one slice of the `talkbank-tools` monorepo:
+
+- `python/batchalign/` — Python package (this file).
+- `crates/batchalign/` — Rust crates (`batchalign-core`, `batchalign-engine`).
+- The pinned TalkBank Chatter dependency supplies CHAT parsing, models, and transforms.
+- `apps/batchalign/batchalign-gui/` — Tauri desktop GUI.
+- `book/` — user + developer documentation (mdBook; source of truth).
+
+For repo conventions, build commands, and the BA3 cutover plan, see
+`CLAUDE.md` at the repo root and
+`book/src/batchalign/developer/landing-status.md`.