diffusers-workflow 0.4.0__py3-none-any.whl

dw/content_types.py

@@ -0,0 +1,150 @@
+"""A step's result 'content_type': the MIME type its writer is chosen by.
+
+`content_type: "video"` validated clean and then died deep inside a writer,
+in a traceback naming neither the field nor the value (#168) - the same
+shape as #162 in a different field. The writer's own dispatch matches on
+`content_type.startswith("video")`, so the bare word "video" took the video
+branch anyway; having no real MIME type it also had no extension to write,
+and the closest writer imageio could guess from an empty one was not a video
+writer at all.
+
+Audio and video each go through exactly one container this engine writes -
+`AUDIO_FORMATS` in `dw/result.py`, and the single `video/mp4` mux - so
+anything else in either family is refused here rather than accepted only to
+mismatch its writer later. image/*, text/* and *.json values stay
+permissive beyond the MIME-shape check: their writer dispatch is a generic
+prefix/suffix match (PIL's own format inference, a literal text or JSON
+write) with no narrower container to enforce a whitelist against.
+
+The one text/* exception is active content. `/outputs` serves a written
+file on the UI's own origin, without a token, so an .html or .xml output is
+a page whose script reads the API token the UI keeps in localStorage (#407).
+`text/html` and `text/xml` are the two active types the text writer can
+produce, so they are refused outright; the server also serves every active
+type it finds on disk under a `Content-Security-Policy: sandbox`, since a
+planted file never passes through here.
+"""
+
+from .for_each import MEMBER_SEPARATOR, render_path
+from .result import AUDIO_FORMATS, MUXED_VIDEO_CONTENT_TYPE
+from .security import InvalidInputError, validate_content_type
+
+CONTENT_TYPE_KEY = "content_type"
+
+# Reference prefixes substitution resolves before this pass runs. One still
+# spelled out here is one nothing resolved, and that is the undeclared-
+# variable pass's complaint rather than a shape error
+_UNRESOLVED_PREFIXES = ("variable:", "item:")
+
+# Result types a browser would run as a document on the UI origin
+REFUSED_ACTIVE_CONTENT_TYPES = frozenset({"text/html", "text/xml"})
+
+
+def _active_content_fault(value):
+    # compared without parameters or case: 'Text/HTML; charset=utf-8' is
+    # the same document type
+    if (
+        isinstance(value, str)
+        and value.split(";", 1)[0].strip().lower() in REFUSED_ACTIVE_CONTENT_TYPES
+    ):
+        return (
+            f"Invalid content_type: {value!r} - active content is not written: "
+            f"a browser would run it as a page on the server's origin. Use "
+            f"'text/plain' or 'application/json'"
+        )
+    return None
+
+
+def content_type_fault(value):
+    """Why this result 'content_type' is invalid, or None.
+
+    Raises nothing - callers that already have an InvalidInputError-raising
+    check (validate_content_type) can call that directly; this is the
+    string-message form `content_type_errors` collects.
+    """
+    try:
+        validate_content_type(value)
+    except InvalidInputError as e:
+        return str(e)
+
+    active = _active_content_fault(value)
+    if active is not None:
+        return active
+    main_type = value.split("/", 1)[0]
+    if main_type == "audio" and value not in AUDIO_FORMATS:
+        return (
+            f"Invalid content_type: {value!r} - audio can be written as "
+            f"{', '.join(sorted(AUDIO_FORMATS))}"
+        )
+    if main_type == "video" and value != MUXED_VIDEO_CONTENT_TYPE:
+        return (
+            f"Invalid content_type: {value!r} - video is only written as "
+            f"'{MUXED_VIDEO_CONTENT_TYPE}'"
+        )
+    return None
+
+
+def content_type_errors(workflow_definition, source_indices=None):
+    """Every result 'content_type' that no writer will accept, as
+    [{path, message}].
+
+    The definition handed here has already been substituted and expanded,
+    so every value in it is literal; a 'variable:' or 'item:' still spelled
+    out is left alone. `source_indices`, when given, is the source step
+    index of each step - a 'for_each' group turns one written step into
+    several, and the path an error carries has to be one the author can
+    find in the file they wrote; the member is named in the message.
+    """
+    steps = workflow_definition.get("steps")
+    if not isinstance(steps, list):
+        return []
+
+    errors = []
+    for index, step in enumerate(steps):
+        if not isinstance(step, dict):
+            continue
+        result = step.get("result")
+        if not isinstance(result, dict) or CONTENT_TYPE_KEY not in result:
+            continue
+        value = result[CONTENT_TYPE_KEY]
+        if isinstance(value, str) and value.startswith(_UNRESOLVED_PREFIXES):
+            continue
+        source = (
+            source_indices[index]
+            if source_indices is not None and index < len(source_indices)
+            else index
+        )
+        name = step.get("name")
+        where = (
+            f" in member '{name}'"
+            if isinstance(name, str) and MEMBER_SEPARATOR in name
+            else ""
+        )
+        fault = content_type_fault(value)
+        if fault is not None:
+            errors.append(
+                {
+                    "path": render_path(("steps", source, "result", CONTENT_TYPE_KEY)),
+                    "message": f"{fault}{where}",
+                }
+            )
+    return errors
+
+
+def refuse_active_content_type(value):
+    """Raise InvalidInputError for an active result type - the writer's
+    run-time half of the refusal `content_type_errors` makes, for a
+    definition that reached it without validation. Only this refusal: the
+    writer's own dispatch still answers every other value as it did."""
+    fault = _active_content_fault(value)
+    if fault is not None:
+        raise InvalidInputError(fault)
+    return value
+
+
+__all__ = [
+    "REFUSED_ACTIVE_CONTENT_TYPES",
+    "content_type_errors",
+    "content_type_fault",
+    "refuse_active_content_type",
+]

dw/dissolve_frame_errors.py

@@ -0,0 +1,121 @@
+"""A `dissolve_videos` overlap too wide for one of its own inputs, refused
+before the run when the frame counts are already knowable.
+
+`dissolve_videos` (`dw/tasks/dissolve_videos.py`) raises once it has decoded
+every input: a video with fewer frames than its share of `dissolve_frames`
+overlaps (`seams * dissolve_frames`) fails with "video N has M frames, too
+few for its S dissolve(s) of F frames". That is correct, but late - a chain
+that generates each shot before joining them can spend many GPU minutes
+reaching a step that was always going to fail, for an arithmetic mistake
+visible from the workflow document alone (#400).
+
+Moved here, into `validation_errors`, for exactly the cases where a video's
+frame count is knowable without running anything: a literal file path inside
+the directories the run may read (`dw/probe_paths.py`), or an
+`asset:`/`output:` reference, with a literal `dissolve_frames`.
+`resolve_path_references` is what turns either into a real path before the
+run reads it; `probe_media` decodes that file the same way `dw/server/app.py`
+already does for gallery metadata. A `previous_result:` (or any reference
+`expand_for_each` left unresolved), a `variable:`/`item:`/`gather:` reference,
+or a non-literal `dissolve_frames`, names no frame count yet and is left to
+the existing run-time check - silence there is correct, not a gap, since the
+length is not known until the step that produces it runs.
+
+`concat_videos`'s `trim_frames` and `crossfade_audio`'s crossfade window were
+each considered for the same treatment - the issue that motivated this module
+asked whether they "probably have the same gap". They do not: neither raises
+when an input is too short. `concat_videos` silently truncates
+(`frames.extend(clip[head_trim:])`), and `crossfade_audio` silently clamps
+its window to the shortest side (`crossfade_concat`) - a different, and
+already silent, shape of problem with no run-time error to move earlier.
+"""
+
+from .for_each import MEMBER_SEPARATOR, render_path
+from .media_info import probe_media
+from .probe_paths import resolve_probe_path
+
+
+def _frame_count(path):
+    """The frame count `dissolve_videos` would see for this file, or None
+    when it cannot be probed or carries no video stream."""
+    info = probe_media(path)
+    if info is None or info.get("kind") != "video":
+        return None
+    return info.get("frame_count")
+
+
+def dissolve_frame_errors(workflow_definition, source_indices=None, base_dir=None):
+    """Every `dissolve_videos` step whose overlap already exceeds a
+    statically-resolvable input's real frame count, as [{path, message}].
+
+    Walks the substituted, expanded definition, the same convention
+    `video_extension_errors` and `task_argument_errors` follow:
+    `source_indices` maps an expanded step back to the one the author wrote,
+    and a path inside a `for_each` member names the member.
+    """
+    steps = workflow_definition.get("steps")
+    if not isinstance(steps, list):
+        return []
+
+    errors = []
+    for index, step in enumerate(steps):
+        if not isinstance(step, dict):
+            continue
+        task = step.get("task")
+        if not isinstance(task, dict) or task.get("command") != "dissolve_videos":
+            continue
+        arguments = task.get("arguments")
+        if not isinstance(arguments, dict):
+            continue
+        videos = arguments.get("videos")
+        if not isinstance(videos, list) or len(videos) < 2:
+            continue
+        dissolve_frames = arguments.get("dissolve_frames", 12)
+        if not isinstance(dissolve_frames, (int, float)) or isinstance(
+            dissolve_frames, bool
+        ):
+            continue
+        if dissolve_frames <= 0:
+            continue
+
+        problems = []
+        for video_index, video in enumerate(videos):
+            path = resolve_probe_path(video, base_dir, "a video argument")
+            if path is None:
+                continue
+            frame_count = _frame_count(path)
+            if frame_count is None:
+                continue
+            seams = (video_index > 0) + (video_index < len(videos) - 1)
+            needed = seams * dissolve_frames
+            if frame_count < needed:
+                problems.append(
+                    f"video {video_index} has {frame_count} frames, too few "
+                    f"for its {seams} dissolve(s) of {dissolve_frames} frames"
+                )
+        if not problems:
+            continue
+
+        source = (
+            source_indices[index]
+            if source_indices is not None and index < len(source_indices)
+            else index
+        )
+        name = step.get("name")
+        where = (
+            f" in member '{name}'"
+            if isinstance(name, str) and MEMBER_SEPARATOR in name
+            else ""
+        )
+        errors.append(
+            {
+                "path": render_path(
+                    ("steps", source, "task", "arguments", "dissolve_frames")
+                ),
+                "message": f"dissolve_videos: {'; '.join(problems)}{where}",
+            }
+        )
+    return errors
+
+
+__all__ = ["dissolve_frame_errors"]

dw/docs/ACCELERATION.md

@@ -0,0 +1,352 @@
+# Inference Acceleration
+
+Speed up generation by caching intermediate computations and skipping redundant transformer steps. Two systems are available: diffusers built-in caching and TeaCache. Beyond caching, `torch.compile`, attention backend selection, layerwise casting, and device-level settings (TF32, cuDNN) also affect throughput - see below. Memory offloading trades speed for VRAM and is covered in depth in [WORKFLOW_GUIDE.md](WORKFLOW_GUIDE.md#memory-offloading).
+
+For ready-made configurations that combine these levers per model family, see [RECIPES_24GB.md](RECIPES_24GB.md).
+
+## Diffusers Built-in Cache
+
+Applied at pipeline load time via the `cache` configuration. Hooks auto-reset between runs.
+
+### Models diffusers has not registered
+
+`first_block`, `mag` and `layer_skip` look a model's transformer block class up in
+diffusers' own registry and raise when it is absent, which is how a model that supports
+`enable_cache()` ends up with no usable cache. [cache_blocks.json](../dw/cache_blocks.json)
+fills those gaps in, registering the missing block metadata on demand; entries become
+redundant, not wrong, once diffusers registers the same class upstream. MiniMax-H3 and
+LTX-2 are listed there today.
+
+LTX-2's block needs one thing more than the metadata diffusers defines. It returns two
+streams - video and audio - and diffusers reads the second one back out of a forward
+argument named literally `encoder_hidden_states`, the only two-stream shape it registers
+upstream (text beside image). LTX-2's blocks take an `encoder_hidden_states` of their
+own, the text conditioning, so the fixed name reads the wrong tensor and feeds the text
+embeddings back as the audio stream on every skipped block. The entry names the argument
+its second stream actually comes from (`encoder_hidden_states_argument_name`), which is
+what makes caching correct there rather than merely quiet.
+
+### FirstBlockCache
+
+Simplest and broadest support. Compares first-block residuals to decide whether to skip remaining blocks.
+
+```json
+"configuration": {
+    "component_type": "FluxPipeline",
+    "cache": {
+        "type": "first_block",
+        "threshold": 0.05
+    }
+}
+```
+
+Higher threshold = more speedup, more quality loss. Start with `0.05` and increase to taste.
+
+**Example:** [step-caching.json](../workflows/templates/step-caching.json)
+
+### MagCache
+
+Magnitude-based caching with error accumulation. Requires `num_inference_steps` to match the pipeline arguments, and `mag_ratios` — the per-step magnitude ratios, which are checkpoint-dependent:
+
+```json
+"cache": {
+    "type": "mag",
+    "mag_ratios": "flux",
+    "threshold": 0.06,
+    "num_inference_steps": 28,
+    "max_skip_steps": 3,
+    "retention_ratio": 0.2
+}
+```
+
+| Property | Default | Description |
+| -------- | ------- | ----------- |
+| `mag_ratios` | required | Preset name or explicit per-step ratio array — see below |
+| `threshold` | 0.06 | Accumulated error threshold for skipping |
+| `num_inference_steps` | required | Must match pipeline arguments |
+| `max_skip_steps` | 3 | Max consecutive steps to skip |
+| `retention_ratio` | 0.2 | Fraction of initial steps where skipping is disabled |
+| `calibrate` | false | Measure ratios for a new model instead of skipping — see below |
+
+#### Supplying `mag_ratios`
+
+MagCache needs to know how each denoising step's output magnitude typically behaves for *your* checkpoint, so unlike the other cache types it cannot run on defaults alone. Give it either:
+
+- **A preset name** — `"mag_ratios": "flux"` resolves to the ratios diffusers ships for Flux. Any preset a later diffusers release adds is usable by name without a change here.
+- **An explicit array** — `"mag_ratios": [1.0, 0.98, 0.96, ...]`. The array is interpolated automatically when its length differs from `num_inference_steps`, so ratios measured at one step count can be reused at another.
+
+For a model with no preset, run once with `"calibrate": true`. Calibration skips nothing and logs the measured ratios at the end of the run; paste that array into `mag_ratios` and drop the `calibrate` flag for subsequent runs.
+
+```json
+"cache": { "type": "mag", "calibrate": true, "num_inference_steps": 28 }
+```
+
+### TaylorSeerCache
+
+Taylor series approximation of cached outputs:
+
+```json
+"cache": {
+    "type": "taylorseer",
+    "cache_interval": 5,
+    "max_order": 1
+}
+```
+
+| Property | Default | Description |
+| -------- | ------- | ----------- |
+| `cache_interval` | 5 | Full computation every N steps |
+| `max_order` | 1 | Taylor series order (higher = better approximation, more memory) |
+
+### FasterCache
+
+Experimental, video-oriented. Uses FFT frequency decomposition:
+
+```json
+"cache": {
+    "type": "faster"
+}
+```
+
+Best for video models like CogVideoX. No additional parameters needed for basic use.
+
+### TextKVCache
+
+Caches the transformer's key/value projections of the (unchanging) text
+embeddings across denoising steps, recomputing only what the latents need:
+
+```json
+"cache": {
+    "type": "text_kv"
+}
+```
+
+No parameters.
+
+## TeaCache
+
+Training-free acceleration that monkey-patches the transformer's forward function. Uses polynomial-rescaled L1 distance to determine when to skip computation.
+
+```json
+"configuration": {
+    "component_type": "FluxPipeline",
+    "teacache": {
+        "rel_l1_thresh": 0.6
+    }
+}
+```
+
+TeaCache requires `num_inference_steps` in the pipeline arguments — it needs to know the total step count.
+
+### Configuration
+
+| Property | Description |
+| -------- | ----------- |
+| `rel_l1_thresh` | Cache threshold. Model-specific defaults apply if omitted. |
+| `coefficients` | Array of 5 polynomial coefficients. Override model defaults. |
+| `variant` | Explicit model variant for multi-variant architectures. |
+
+### Supported Models
+
+Model coefficients and defaults are stored in [teacache_models.json](../dw/teacache_models.json). Currently implemented with a custom forward function:
+
+- **Flux** (FluxTransformer2DModel) — thresholds: 0.25 (~1.5x), 0.4 (~1.8x), 0.6 (~2.0x), 0.8 (~2.25x)
+
+Registry includes coefficients for Mochi, LTX-Video, CogVideoX, HunyuanVideo, Wan2.1, and Lumina2 (forward functions pending). For any model other than Flux, use the [diffusers built-in caches](#diffusers-built-in-cache) instead - `first_block` or `mag` cover the models the registry lists.
+
+### Variants
+
+Some models have multiple variants with different coefficients:
+
+```json
+"teacache": {
+    "rel_l1_thresh": 0.2,
+    "variant": "cogvideox_2b"
+}
+```
+
+**Example:** [step-caching.json](../workflows/templates/step-caching.json)
+
+## Cache vs TeaCache
+
+| | Diffusers Cache | TeaCache |
+| --- | --- | --- |
+| Setup | Built into diffusers | Custom forward functions |
+| Model support | Any transformer with CacheMixin | Requires per-model implementation |
+| Maintenance | Maintained by HuggingFace | Maintained in this project |
+| Configuration | Set once at load time | Applied per-execution via context manager |
+| Approach | Various algorithms (block, magnitude, Taylor) | Polynomial-rescaled L1 distance |
+
+They are **mutually exclusive** — use one or the other, not both.
+
+For most cases, start with `first_block` cache. Use TeaCache when you need fine-tuned control over Flux acceleration thresholds.
+
+## Attention Backends
+
+Select the attention implementation diffusers uses for the duration of each pipeline call, via a context manager wrapped around `pipeline(...)`:
+
+```json
+"configuration": {
+    "component_type": "FluxPipeline",
+    "attention_backend": "flash_hub"
+}
+```
+
+Common values: `"flash"`, `"flash_hub"`, `"sage"`, `"sage_hub"`, `"native"`, `"flex"`. The full set is diffusers' `AttentionBackendName` enum - availability depends on what's installed (`flash-attn`, `sageattention`, etc.) and the platform. `_hub`-suffixed backends are fetched from the Hugging Face Hub kernel registry on first use, which needs the `kernels` package installed (`pip install kernels`) - it is not a dw dependency, and no bundled workflow sets a backend, so each runs on a plain install.
+
+A component can also pin its backend persistently instead, via `set_attention_backend`:
+
+```json
+"configuration": {
+    "components": {
+        "transformer": { "attention_backend": "flash_hub" }
+    }
+}
+```
+
+Prefer the pinned form for a compiled component - the per-call context manager switches implementations under the compiled graph and forces a recompile on every run.
+
+## Attention Slicing
+
+```json
+"configuration": {
+    "component_type": "FluxPipeline",
+    "enable_attention_slicing": true
+}
+```
+
+Processes attention in slices to reduce memory at some cost to speed. Enabled automatically on MPS (unified memory benefits from slicing) unless `disable_attention_slicing` is set. Modular pipelines have no `enable_attention_slicing()` method - the setting is silently skipped rather than failing when the pipeline doesn't support it.
+
+## torch.compile
+
+Compile a component once it is fully configured - the graph captures final dtypes, adapters, quantization, and offload hooks. Configured per component under `components`:
+
+```json
+"configuration": {
+    "component_type": "FluxPipeline",
+    "components": {
+        "transformer": {
+            "compile": {
+                "repeated_blocks": true,
+                "fullgraph": true
+            }
+        }
+    }
+}
+```
+
+| Property | Description |
+| -------- | ----------- |
+| `repeated_blocks` | Compile only the model's repeated block classes (diffusers regional compilation). Near the same speedup as full compilation with a fraction of the cold-start cost. Recommended. |
+| `mode` | torch.compile mode: `"default"`, `"reduce-overhead"`, `"max-autotune"`. |
+| `fullgraph` | Require a single graph with no breaks - fails fast instead of silently losing speedup. |
+| `dynamic` | Compile with dynamic shapes. Set `true` when resolutions or frame counts vary between runs to avoid recompiles. |
+
+Typical gains are 1.3-1.5x on diffusion transformers, and compilation stacks with the caches above. Notes:
+
+- **First run pays the compile cost.** The [REPL](REPL_COMMANDS.md)'s persistent worker keeps compiled pipelines loaded between runs, so the cost is paid once per session rather than once per generation.
+- **Pin the attention backend** on a compiled component (`"attention_backend"` in the same `components` entry) rather than using the pipeline-level per-call context manager, which forces recompiles.
+- **Composes with offloading**: apply `group_offload` and `compile` on the same component and the offload hooks are installed first, as required. Skipped with a warning on MPS.
+- **Don't combine `fullgraph` with a `cache`**: the cache hooks decide skip-or-compute per step, a data-dependent branch diffusers wraps in `torch.compiler.disable` - it needs the graph break that `fullgraph: true` forbids. Compile with the default (partial) graph mode when a cache is active.
+- **TorchAO quantization needs compile to be fast** - see [QUANTIZATION.md](QUANTIZATION.md#torchao).
+
+**Example:** [flux-dev-compile.json](../workflows/models/flux-dev-compile.json), [flux-torchao.json](../workflows/models/flux-torchao.json)
+
+## Layerwise Casting
+
+Store a component's weights in a narrow dtype and upcast only for compute, per component:
+
+```json
+"transformer": {
+    "configuration": { "component_type": "FluxTransformer2DModel" },
+    "enable_layerwise_casting": {
+        "storage_dtype": "torch.float8_e4m3fn",
+        "compute_dtype": "torch.bfloat16"
+    },
+    "from_pretrained_arguments": { ... }
+}
+```
+
+Both `storage_dtype` and `compute_dtype` are required. Applied via the component's own `enable_layerwise_casting()` right after it loads, so it composes with quantization and group offloading on the same component.
+
+## Memory Format and Attention Processors
+
+Three older per-component knobs, set in the pipeline `configuration` beside
+`vae` / `unet` / `transformer` and applied right after the components load:
+
+| Key | Where | Effect |
+| --- | ----- | ------ |
+| `channels_last` | `vae`, `unet` | `to(memory_format=torch.channels_last)`. Faster convolutions on CUDA for a convolutional UNet or VAE; nothing to gain on a transformer |
+| `enable_forward_chunking` | `unet` | Runs the UNet's feed-forward layers in chunks - less peak memory, slightly slower |
+| `attn_processor_type` | `unet`, `transformer` | Names an attention processor class to install with `set_attn_processor` (the name is resolved and constructed, so it goes through the `_type` conversion: `"AttnProcessor2_0"`). For a per-call backend instead, see [Attention Backends](#attention-backends) |
+
+```json
+"configuration": {
+    "component_type": "StableDiffusionPipeline",
+    "unet": { "channels_last": true, "enable_forward_chunking": true },
+    "vae": { "enable_slicing": true, "channels_last": true }
+}
+```
+
+## Memory Offloading
+
+`offload` (`"model"` or `"sequential"`) and `group_offload` trade speed for VRAM by streaming weights between system memory and the accelerator instead of keeping everything resident. `"model"` moves whole submodules and costs the least speed; `"sequential"` moves individual layers and is the slowest but uses the least memory; block/leaf-level `group_offload` sits between the two and is what a modular pipeline's self-loaded components use, since they aren't reachable in time for `offload`. Full configuration syntax is in [WORKFLOW_GUIDE.md](WORKFLOW_GUIDE.md#memory-offloading). Omit both for the fastest run, when VRAM allows it.
+
+`"residency": "on_demand"` on a component is the cheap case of the same trade: the model rests in system memory and is moved to the device whole around each of its own calls. That is a bad deal for anything called once per step, and a good one for a VAE called twice a run - it frees the VAE's VRAM for the denoise loop at the cost of two transfers, where group offloading the same VAE would restream it once per decode tile. See [On-demand components](WORKFLOW_GUIDE.md#on-demand-components).
+
+**Example:** [flux-dev.json](../workflows/models/flux-dev.json) (`"offload": "model"`), [z-image.json](../workflows/models/z-image.json) (`"offload": "sequential"`), [video-with-audio.json](../workflows/templates/minimax/video-with-audio.json) (`group_offload` per component), [reference-to-video.json](../workflows/templates/minimax/reference-to-video.json) (`group_offload` for the transformer, `on_demand` for the VAEs)
+
+## Reading Memory While Offloading
+
+A workflow that offloads keeps its weights in host memory by design, so the
+card can sit near-empty through a generation and the VRAM figures alone say
+nothing about what a run holds or fails to release. `get_memory` (MCP) and
+`GET /api/memory` report both: `gpu_*` is the card, `host_memory_rss_mb` is
+what the worker process holds and `host_memory_peak_rss_mb` the most it has
+ever held, beside the machine's `host_memory_total_mb` /
+`host_memory_available_mb`.
+
+`host_pinned_reserved_mb` / `host_pinned_allocated_mb`, where the platform
+reports them, are torch's pinned-host cache - the staging buffers group
+offloading moves weights through. They are part of `host_memory_rss_mb` and
+invisible in every `gpu_*` figure, so a worker that has released every model
+and still holds gigabytes is usually holding these; they are returned when
+the worker switches to a different workflow (#98). A host field is absent,
+rather than null, on a platform that cannot measure it.
+
+## TF32 and cuDNN
+
+Device-level settings, read once at startup from `~/.diffusers_helper/settings.json`:
+
+| Setting | Default | Effect |
+| ------- | ------- | ------ |
+| `enable_tf32` | `true` | Sets `torch.set_float32_matmul_precision("high")`, and on CUDA also `torch.backends.cuda.matmul.allow_tf32 = True`. ~2x faster matmuls on Ampere+ GPUs (RTX 30/40 series, A100, H100) with minor precision loss. No effect outside CUDA. |
+| `cudnn_benchmark` | `true` | CUDA only. Autotunes cuDNN algorithm selection - fastest for a workflow with fixed input sizes, can add overhead when sizes vary run to run. |
+| `cudnn_deterministic` | `false` | CUDA only. Set `true` to trade speed for reproducible output given the same seed. |
+
+```json
+{ "enable_tf32": true, "cudnn_benchmark": true, "cudnn_deterministic": false }
+```
+
+## Environment Defaults
+
+Set automatically at import unless already present in the environment (export your own value to override):
+
+| Variable | Default | Effect |
+| -------- | ------- | ------ |
+| `PYTORCH_CUDA_ALLOC_CONF` | `expandable_segments:True` | Lets the CUDA allocator grow segments instead of fragmenting fixed-size ones. Multi-step workflows churn differently-shaped allocations (generate, upscale, interpolate); fragmentation is what OOMs a card that nominally has room. |
+| `HF_ENABLE_PARALLEL_LOADING` | `true` | Loads sharded checkpoints in parallel - faster cold starts. |
+| `PYTORCH_MPS_HIGH_WATERMARK_RATIO` | `0.0` | MPS only - use all available unified memory. |
+
+For faster model downloads, optionally `pip install hf_transfer` and set `HF_HUB_ENABLE_HF_TRANSFER=1`. Not enabled automatically - it bypasses the Python HTTP stack and breaks some proxy setups.
+
+## MPS Notes
+
+Apple Silicon has narrower acceleration support than CUDA:
+
+- No flash-attn, no Triton, no bitsandbytes - `attention_backend` is effectively CUDA-only; use `"native"`-family backends or leave it unset on MPS. `compile` is skipped with a warning (inductor support on MPS is immature).
+- No `torch.autocast` support - autocast-related warnings from other libraries are suppressed automatically rather than surfaced.
+- `enable_attention_slicing` is on by default (set `disable_attention_slicing` to turn it off).
+- `float16` produces NaN values on Apple Silicon - use `float32` or `bfloat16` for `torch_dtype` instead; dw only warns, it doesn't override the dtype for you.
+- `PYTORCH_MPS_HIGH_WATERMARK_RATIO` defaults to `0.0` (use all unified memory) unless already set in the environment.
+- Offloading has less benefit than on CUDA, since unified memory is already shared between CPU and GPU.