marshmallow-core 0.1.0__tar.gz

marshmallow_core-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,44 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: test (py${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Build the wheel (Rust core + Python) and install it
+        run: |
+          python -m pip install --upgrade pip maturin pytest
+          maturin build --release --out dist
+          # Pulls stock marshmallow from PyPI as a declared dependency.
+          python -m pip install dist/*.whl
+
+      - name: Run the suite (core active)
+        run: pytest -q
+
+      - name: Run the suite with the core disabled
+        env:
+          MARSHMALLOW_NO_ACCEL: "1"
+        # The handshake tests assert the core is active, so exclude them here;
+        # everything else must still pass on the pure-Python fallback.
+        run: pytest -q -k "not core_active and not protocol"

marshmallow_core-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,78 @@
+name: Release
+
+# Builds abi3 wheels + sdist on a version tag and publishes them to PyPI via
+# trusted publishing. Setup required once before the first release:
+#   1. Create the PyPI project's trusted publisher pointing at this repo +
+#      workflow (https://docs.pypi.org/trusted-publishers/).
+#   2. Create a GitHub Environment named "pypi" (Settings > Environments).
+# Then push a tag like `v0.1.0` (or run manually for a dry run that only builds).
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  wheels:
+    name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
+    runs-on: ${{ matrix.platform.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - { runner: ubuntu-latest, target: x86_64, manylinux: auto }
+          - { runner: ubuntu-latest, target: aarch64, manylinux: auto }
+          # No Intel macOS wheel: the scarce macos-13 (Intel) runners queue for a
+          # long time and are being wound down, so they blocked the release. Only
+          # Apple-Silicon (arm64) macOS wheels are built; Intel-Mac users install
+          # from the sdist (which compiles the Rust core locally).
+          - { runner: macos-14, target: aarch64, manylinux: "" }
+          - { runner: windows-latest, target: x64, manylinux: "" }
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build abi3 wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.platform.target }}
+          args: --release --out dist
+          manylinux: ${{ matrix.platform.manylinux }}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }}
+          path: dist
+
+  sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build sdist
+        uses: PyO3/maturin-action@v1
+        with:
+          command: sdist
+          args: --out dist
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-sdist
+          path: dist
+
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [wheels, sdist]
+    if: startsWith(github.ref, 'refs/tags/')
+    environment: pypi
+    permissions:
+      id-token: write # trusted publishing; no API token needed
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+      - uses: pypa/gh-action-pypi-publish@release/v1

marshmallow_core-0.1.0/.gitignore

@@ -0,0 +1,16 @@
+# Rust
+/target
+**/*.rs.bk
+
+# Python
+__pycache__/
+*.py[cod]
+*.so
+*.pyd
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/

marshmallow_core-0.1.0/BACKLOG.md

@@ -0,0 +1,81 @@
+# Backlog
+
+Flat, actionable todo list derived from [ROADMAP.md](ROADMAP.md). Work top to
+bottom within a phase.
+
+## Phase 0 — Measurement
+
+- [x] Create `performance/` directory
+- [x] Port `benchmark.py` from the fork; drive it via `install()`/`uninstall()`
+- [x] Add flat-scalar schema benchmark case
+- [x] Add nested schema benchmark case
+- [x] Add list-heavy schema benchmark case
+- [x] Add validator-heavy schema benchmark case
+- [x] Benchmark `dump`, `load`, `dumps`, `loads` for each case
+- [x] Print a stock-vs-core comparison table
+- [x] Write a coverage probe: report per-field native vs callback for a schema
+- [x] Document how to run the benchmark in README
+
+## Phase 1 — Coverage gaps
+
+### Hook-bearing load schemas
+- [x] Map out marshmallow 4.x `_do_load` body (pre_load → deserialize → validators → post_load)
+- [x] In `_patched_do_load`, run `pre_load` in Python before the core step
+- [x] Call the core for the per-field deserialize when hooks are present
+- [x] Run field validators in Python after the core step
+- [x] Run schema validators (`validates_schema`) in Python
+- [x] Run `post_load` in Python after the core step
+- [x] Remove the `not _has_load_hooks(self)` guard once the split works
+- [x] Add equivalence tests for pre_load + post_load + validates schemas
+
+### Native validators
+- [x] Add a validator tag space (compiler + lib.rs)
+- [x] Native `Range` validator
+- [x] Native `Length` validator
+- [x] Native `OneOf` validator
+- [x] Compile only recognized validators; fall back on any other
+- [x] Equivalence tests: pass + fail inputs for each validator
+
+### New field types
+- [x] Native `Decimal` (dump + load)
+- [x] Native `Dict`/`Mapping` (dump + load) — plain dict-copy case
+- [x] Native `Constant`
+- [x] Share `Number` base coercion with Integer/Float — Decimal reuses the
+      field's own (`Number`-based) `_serialize`/`_deserialize` rather than
+      duplicating coercion in Rust
+- [x] Equivalence tests for each new field type (valid + error)
+
+## Phase 2 — Fused JSON path
+
+- [x] Add a dump→JSON-bytes core object in `lib.rs` (`DumpSerializer.run_json`)
+- [x] Build the JSON-dump payload in `_compiler.py` (`build_dump_json_serializer`)
+- [x] Wrap `Schema.dumps` in `_patch.py`
+- [x] Equivalence tests: `dumps` output matches stock
+- [~] Add a JSON→loaded core object in `lib.rs` — **implemented + benchmarked,
+      not shipped.** A `serde_json`-backed single-pass `LoadDeserializer.run_json`
+      was prototyped, but it is consistently *slower* than CPython's C `json.loads`
+      followed by the already-accelerated `_do_load` (e.g. nested 2.12µs vs 1.96µs;
+      100-record list 60µs vs 37µs). `loads` already benefits from the `_do_load`
+      patch, so fusing it only adds a heavy dependency and a regression. Reverted
+      per the roadmap's "confirm a real gain, not a regression" rule.
+- [~] Build the JSON-load payload in `_compiler.py` — see above (reverted)
+- [~] Wrap `Schema.loads` in `_patch.py` with `AccelFallback` — see above (reverted)
+- [~] Equivalence tests: `loads` result + errors match stock — verified during the
+      prototype (big-int/error/unknown-key parity held); dropped with the revert
+
+## Phase 3 — Remaining deferrals + micro-opts
+
+- [x] Support `unknown=INCLUDE`
+- [x] Support collection/dotted `partial`
+- [x] Support dotted attribute writes (`set_value`)
+- [x] Cache `_has_load_hooks` on the class (per-class `WeakKeyDictionary`)
+- [x] Precompute the known-keys frozenset once (built per compile, held by the
+      Rust `LoadSerializer`)
+
+## Per-change checklist (every code item)
+
+- [x] Implemented in **both** `_compiler.py` and `lib.rs`
+- [x] Dump/load tags kept in sync
+- [x] Bumped `PROTOCOL_VERSION` + `_EXPECTED_PROTOCOL` if payload/tags changed
+- [x] Equivalence test added (valid + error inputs)
+- [x] Re-ran benchmark to confirm a real gain

marshmallow_core-0.1.0/CLAUDE.md

@@ -0,0 +1,145 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## What this is
+
+**marshmallow_core** is a standalone, opt-in Rust acceleration core for the
+published [marshmallow](https://github.com/marshmallow-code/marshmallow) library.
+It installs *alongside* an unmodified `pip install marshmallow` and, when
+activated, monkey-patches `Schema._serialize` / `Schema._do_load` to run a PyO3
+extension instead of marshmallow's per-object Python loops. It is **strictly a
+speedup**: with the core installed, `dump`/`load` produce results identical to
+stock marshmallow, falling back to pure Python for anything not modelled
+natively (and, on load, for *every* error/edge case).
+
+Activation is explicit and process-wide:
+
+```python
+import marshmallow_core
+marshmallow_core.install()      # patch marshmallow.Schema
+...                             # use marshmallow as usual
+marshmallow_core.uninstall()    # restore the stock methods
+```
+
+This is a mixed Python/Rust project built with **maturin** (`flit`-style src
+layout under `python/`, Rust under `src/`). Python 3.10+ ; the extension is
+`abi3` so one wheel covers all supported versions.
+
+## Provenance
+
+The Rust core (`src/lib.rs`) and the compiler (`python/marshmallow_core/_compiler.py`)
+were lifted from marshmallow's own `rust-core` branch — `crates/marshmallow-core/src/lib.rs`
+and `src/marshmallow/_accel.py` respectively. The crucial difference: in that
+branch the core is wired *into* `schema.py`; here it lives outside marshmallow
+and patches in from the outside via `_patch.py`. When fixing core logic, the
+upstream branch is the reference implementation to diff against.
+
+## Commands
+
+Requires `cargo` (rustup) and [`maturin`](https://www.maturin.rs/) (run via `uvx`).
+
+```bash
+# Build the Rust core + install the package into the current venv (iterating):
+uvx maturin develop --release
+
+# Build a wheel without installing (verifies Rust compiles + maturin metadata):
+uvx maturin build --release            # -> target/wheels/marshmallow_core-*.whl
+
+# Run the suite. IMPORTANT: test against STOCK marshmallow, not a checkout of
+# the fork (the fork has its own baked-in accel and would double-patch). Use a
+# throwaway venv with marshmallow from PyPI:
+WHEEL=$(ls -t target/wheels/*.whl | head -1)
+uv venv /tmp/mc && uv pip install --python /tmp/mc/bin/python marshmallow pytest "$WHEEL"
+/tmp/mc/bin/python -m pytest -q                       # whole suite
+/tmp/mc/bin/python -m pytest tests/test_equivalence.py::test_load_equivalence -q   # one test
+MARSHMALLOW_NO_ACCEL=1 /tmp/mc/bin/python -m pytest -q -k "not core_active and not protocol"  # pure-Python path
+```
+
+`MARSHMALLOW_NO_ACCEL=1` disables the core even after `install()` (the
+pure-Python path is always correct). CI (`.github/workflows/ci.yml`) runs the
+suite on py3.10–3.13 both with the core active and disabled.
+
+## Architecture
+
+Three Python modules plus one Rust file. Read `_patch.py`, then `_compiler.py`,
+then `src/lib.rs`.
+
+### `_patch.py` — the install() shim (replaces the fork's `schema.py` edits)
+
+`install()` saves and overwrites `Schema._serialize` and `Schema._do_load`.
+The wrappers try the compiled core and otherwise call the saved originals:
+
+- The compiled serializer/deserializer is **cached per Schema instance** on
+  `vars(schema)["_mc_dump_serializer"]` / `["_mc_load_deserializer"]` — `_UNSET`
+  (not built), `None` (not compilable → always pure Python), or a core object.
+- `_do_load` only attempts the core when the call uses the schema's own
+  `unknown`, `partial` is `True`-or-falsy, and the schema has **no load hooks**
+  (`_has_load_hooks` checks `pre_load`/`post_load`/`validates`/`validates_schema`).
+  On `AccelFallback` from the core it falls through to the original `_do_load`.
+
+**Key divergence from the upstream branch:** there, a *root* schema with load
+hooks still uses the core for the per-field step with Python running the hooks
+*around* it. That split lives *inside* `_do_load` and cannot be reproduced by
+wrapping the method from outside, so here **hook-bearing schemas use the
+pure-Python load path entirely**. Correct, just not accelerated. Dump is fully
+accelerated regardless of dump hooks (they run via `dump`, not `_serialize`).
+
+### `_compiler.py` — Schema → payload (the "compiler")
+
+Inspects a *bound* Schema and compiles it into a recursive tuple "payload"
+describing each field as either **native** (formatted/parsed entirely in Rust)
+or a **callback** (defers to the Python `Field.serialize`/`deserialize`).
+`build_dump_serializer` / `build_load_deserializer` return `None` to mean "use
+pure Python". `is_available()` gates everything (extension importable + protocol
+match + not `MARSHMALLOW_NO_ACCEL`). It reads only attributes present in stock
+marshmallow 4.x.
+
+### `src/lib.rs` — the PyO3 core
+
+Parses payloads into `DumpSerializer` / `LoadDeserializer`, built as the
+extension `marshmallow_core._core` (`[lib] name = "_core"`, `module-name =
+"marshmallow_core._core"`). The **load** core handles only the happy path: the
+instant it hits any error/edge case it raises `AccelFallback` and Python re-runs
+the unchanged pure-Python load, so every message/value matches exactly.
+`KeyboardInterrupt`/`SystemExit` from a callback propagate unchanged (never
+converted to `AccelFallback`). The **dump core has no `AccelFallback`** — it
+can't defer mid-serialization — so every native dump element must be provably
+identical to `Field._serialize`.
+
+### Invariants when changing the core
+
+- **Dump and load tag spaces are distinct integers** and must stay in sync
+  between `_compiler.py` and `lib.rs`. When adding a native field type, add it
+  in *both* (build the element tuple in `_compiler.py`, parse + apply in
+  `lib.rs`), keep tags aligned, and verify accel-on output equals accel-off
+  output across **valid and error** inputs (`tests/test_equivalence.py`).
+- The extension exports `PROTOCOL_VERSION`; `_compiler._EXPECTED_PROTOCOL` must
+  match it. Bump both together when payload shapes/tags change.
+- **What stays pure-Python:** custom `dict_class`/`get_attribute`,
+  self-referential schemas, custom strptime temporal formats,
+  `NaiveDateTime`/`AwareDateTime` on load, callable defaults, unrecognized field
+  validators (anything but `Range`/`Length`/`OneOf`), field-level
+  `pre_load`/`post_load`, a `Dict` with key/value fields, and any field type
+  without a native element. **Now accelerated** (Phase 1/3): `unknown=INCLUDE`,
+  collection/dotted `partial`, dotted attribute writes (`set_value`),
+  `Range`/`Length`/`OneOf` validators, `Decimal`/`Dict`/`Constant` fields,
+  schema-level load hooks (`pre_load`/`post_load`/`validates`/`validates_schema`
+  run in Python around the core's per-field step), and `dumps` (fused to JSON in
+  Rust). `loads` is accelerated via the patched load path (a fused JSON *parser*
+  was prototyped but did not beat C `json.loads`, so it ships unfused).
+
+## Testing conventions
+
+`tests/test_equivalence.py` is the source of truth: an autouse fixture calls
+`install()`, and each `_dump_both`/`_load_both` helper runs once accelerated,
+then `monkeypatch`es `_compiler.build_*` to `None` to force the pure path, and
+asserts the two are equal. `tests/test_smoke.py` is a quick install/uninstall +
+equivalence sanity check. New native fields need a case in `test_equivalence.py`.
+
+## Development
+
+ - commit only after passing tests
+ - for every big part create a new branch 
+ - for every small task(feature) make a commit
+ - do not commit thing that i'nt make functionality

marshmallow_core-0.1.0/Cargo.lock

@@ -0,0 +1,172 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "indoc"
+version = "2.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "marshmallow_core"
+version = "0.1.0"
+dependencies = [
+ "pyo3",
+]
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "portable-atomic"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d"
+dependencies = [
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6"
+dependencies = [
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.13.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "unindent"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3"

marshmallow_core-0.1.0/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "marshmallow_core"
+version = "0.1.0"
+edition = "2021"
+description = "Rust acceleration core for marshmallow serialization (separate, opt-in package)"
+license = "MIT"
+repository = "https://github.com/gunlinux/marshmallow_core"
+readme = "README.md"
+
+[lib]
+name = "_core"
+crate-type = ["cdylib"]
+
+[dependencies]
+pyo3 = { version = "0.27", features = ["extension-module", "abi3-py310"] }
+
+[profile.release]
+lto = "thin"