freesolo-flash-dev 1.0.97__tar.gz → 1.0.99__tar.gz

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 1.0.97
+Version: 1.0.99
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/flash/engine/worker/backend_common.py

@@ -21,9 +21,12 @@ import re
 import shutil
 import signal
 import subprocess
+import threading
 import time
+import weakref
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import thread as _thread_module
 from http.server import ThreadingHTTPServer
 
 from flash.diagnostics import sanitize_diagnostic
@@ -192,6 +195,49 @@ def agent_loop_workers(rollout_batch: int, *, cap: int = 8) -> int:
 _BRIDGE_WORKER_THREADS = 16
 
 
+class _DaemonBridgeThreadPool(ThreadPoolExecutor):
+    """A ``ThreadPoolExecutor`` whose workers cannot hold the interpreter open.
+
+    The stock executor registers every worker in ``concurrent.futures.thread._threads_queues``,
+    which an interpreter-exit hook joins. ``shutdown(wait=False, cancel_futures=True)`` does not
+    change that: cancelling drops only the QUEUED futures, and a handler already running keeps its
+    thread alive, so exit blocks on it. Measured: a 30-second handler delays exit by the full 30
+    seconds even after ``shutdown(wait=False)`` returns.
+
+    That is the opposite of what a bridge needs. A hung environment reward or teacher callback must
+    not stop the worker from publishing its terminal result and releasing a paid GPU -- and it is
+    also the property the old daemon ``ThreadingHTTPServer`` had before requests moved onto a pool.
+
+    So: start the workers as daemons and skip the exit-hook registration. Only the thread creation
+    differs from the stdlib; queueing, idle reuse, and shutdown are inherited unchanged.
+    """
+
+    def _adjust_thread_count(self) -> None:
+        if self._idle_semaphore.acquire(timeout=0):
+            return
+
+        def weakref_cb(_, q=self._work_queue):
+            q.put(None)
+
+        num_threads = len(self._threads)
+        if num_threads < self._max_workers:
+            thread = threading.Thread(
+                name=f"{self._thread_name_prefix or self}_{num_threads}",
+                target=_thread_module._worker,
+                args=(
+                    weakref.ref(self, weakref_cb),
+                    self._work_queue,
+                    self._initializer,
+                    self._initargs,
+                ),
+                daemon=True,
+            )
+            thread.start()
+            self._threads.add(thread)
+            # deliberately NOT added to _thread_module._threads_queues: that mapping is exactly
+            # what the interpreter-exit hook walks to join workers.
+
+
 class BoundedThreadingHTTPServer(ThreadingHTTPServer):
     """an http server that serves requests from a fixed thread pool instead of one thread each.
 
@@ -212,13 +258,16 @@ class BoundedThreadingHTTPServer(ThreadingHTTPServer):
     """
 
     # daemon threads so a stuck handler can never keep the worker process alive at shutdown.
+    # `daemon_threads` is a ThreadingMixIn flag, read only by the `process_request` this class
+    # overrides, so it does nothing here on its own -- _DaemonBridgeThreadPool below is what
+    # actually delivers the property it names.
     daemon_threads = True
 
     # measured: at the socketserver default of 5, 13 of 64 simultaneous callers were reset by peer.
     request_queue_size = 128
 
     def __init__(self, *args, worker_threads: int = _BRIDGE_WORKER_THREADS, **kwargs):
-        self._bridge_pool = ThreadPoolExecutor(
+        self._bridge_pool = _DaemonBridgeThreadPool(
             max_workers=worker_threads, thread_name_prefix="flash-bridge"
         )
         super().__init__(*args, **kwargs)
@@ -438,8 +487,31 @@ if %(gdn_module)r:
                     torch.zeros(4, 3, device="cuda", dtype=torch.bfloat16),
                 )
                 torch.cuda.synchronize()
+        if not ok:
+            # say WHY on a clean negative too: which of the three checks failed is the difference
+            # between "install fla" and "rebuild causal_conv1d for this arch".
+            print(
+                "[verl] gdn boundary resets unavailable in the child: fla="
+                + str(is_flash_linear_attention_available())
+                + " causal_conv1d="
+                + str(is_causal_conv1d_available()),
+                flush=True,
+            )
         emit("gdn_boundary_resets", bool(ok))
-    except Exception:
+    except Exception as e:
+        # a bare False here is what made this invisible. the caller raises on a hybrid that cannot
+        # reset, and an operator reading only that has no way back to which check fell over --
+        # a missing wheel, a broken ABI, or a conv kernel compiled without this arch all look
+        # identical from the parent. printed to STDOUT, which probe_verl_capabilities forwards
+        # on the '[verl] ' prefix -- stderr would be dropped, it is only reported when the
+        # child answers nothing at all.
+        import traceback
+
+        print(
+            "[verl] gdn boundary resets unavailable in the child: "
+            + " | ".join(traceback.format_exc(limit=3).strip().splitlines()),
+            flush=True,
+        )
         emit("gdn_boundary_resets", False)
 """
 
@@ -523,6 +595,14 @@ def probe_verl_capabilities(python_bin: str, gdn_module: str = "") -> dict:
                 # fail-closed default rather than vanishing from the dict.
                 caps.update({k: v for k, v in answered.items() if k in caps})
                 answered_any = True
+    # the child's own diagnostics, forwarded BEFORE the early return. the gdn question prints why it
+    # answered no, and it is answered LAST -- so every real run answers something earlier, and a
+    # bare `return caps` here would discard the one line the gate's raise tells operators to read.
+    # matched on the prefix rather than forwarded wholesale: the child's stdout also carries verl's
+    # own import chatter, which is not worth a parent log line.
+    for line in (stdout or "").splitlines():
+        if line.startswith("[verl] "):
+            print(line, flush=True)
     if answered_any:
         return caps
     if not timed_out:
@@ -609,6 +689,52 @@ def gdn_reset_arch_from_caps(caps: dict, gdn_module: str) -> str | None:
     return gdn_module.split(".")[2]
 
 
+def require_gdn_boundary_resets(caps: dict, gdn_module: str) -> str | None:
+    """The arch to patch for a gdn hybrid, raising if the child cannot honor boundary resets.
+
+    A thin gate over ``gdn_reset_arch_from_caps``: same answer for the two cases that have one, but
+    a raise where that returns None for a gdn model. Returns None only for a NON-gdn model, which
+    is what an empty ``gdn_module`` means -- the parent never asked the child, because there was
+    nothing to ask about.
+
+    There is no third outcome for a gdn hybrid. One whose child lacks fla/causal_conv1d used to fall
+    back to verl's padded path (``use_remove_padding=False``), which is boundary-correct but cannot
+    complete a single training step on grpo or opd: flash sets ``use_fused_kernels=True``
+    unconditionally, and verl's fsdp engine has no guard for that pair (its megatron engine does,
+    ``megatron/transformer_impl.py:869-874``). The padded+fused branch returns a dense
+    ``[bsz, response_len]`` where every sibling path re-nests via ``cu_seqlens``, so
+    ``no_padding_2_padding`` asserts ``sequence_offsets[-1] == values.shape[0]`` -- total tokens
+    against batch size, which fails at EVERY batch size. Measured on four matched real-gpu arms, all
+    four dead at ``padding.py:144``.
+
+    So the run cannot finish, and the failure names neither gdn nor this decision -- letting it
+    launch spends the full gpu rental on an untraceable shape error. Raise here instead. Dropping
+    ``use_fused_kernels`` is NOT the alternative fix: ``engine/vram.py:386-392`` sizes on the
+    assumption that no dense ``[b, s, vocab]`` logits tensor exists (~130 GB at 32k on a 248k
+    vocab), so that would trade this crash for an OOM. The fix is to give the child the kernels.
+
+    SFT is deliberately not a caller. It reaches the same padded+fused branch but is safe without
+    resets for a different reason: every gdn model profiles as exact-unpacked, which pins
+    ``examples_per_update`` to 1 (``sft_workload.py:410``), so there are no packed neighbours to
+    contaminate -- and its remove-padding flag stays on regardless, because that is the only tensor
+    layout ``FlashTokenizedSFTDataset`` fits. Grpo and opd have no such profile: their batch size is
+    the user's ``prompts_per_step``, routinely greater than 1, so they genuinely pack.
+    """
+    if not gdn_module:
+        return None
+    arch = gdn_reset_arch_from_caps(caps, gdn_module)
+    if arch is None:
+        raise RuntimeError(
+            "gdn hybrid without child-side boundary resets: the padded fallback "
+            "(use_remove_padding=False) is incompatible with the use_fused_kernels=True this "
+            "recipe sets, and cannot complete a training step on verl's fsdp engine. see the "
+            "'[verl] gdn boundary resets unavailable' line above for why the child could not "
+            "honor resets -- installing fla + causal_conv1d in the verl interpreter is the fix, "
+            "not disabling fused kernels."
+        )
+    return arch
+
+
 def resolve_verl_python(workdir: str, *, install_wandb: bool = False) -> str:
     """return an interpreter that can import verl.
 

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/flash/engine/worker/opd_train.py

@@ -38,7 +38,6 @@ from flash.engine.worker.backend_common import (
     agent_loop_workers,
     clamp_engine_len,
     gdn_probe_module,
-    gdn_reset_arch_from_caps,
     latest_global_step_dir,
     model_max_position_embeddings,
     parse_verl_metric,
@@ -48,6 +47,7 @@ from flash.engine.worker.backend_common import (
     render_gdn_varlen_shim,
     render_tf32_shim,
     render_wandb_link_shim,
+    require_gdn_boundary_resets,
     resolve_blackwell_attention_backends,
     resolve_rollout_enforce_eager,
     resolve_verl_loggers,
@@ -1551,9 +1551,10 @@ def build_opd_overrides(config: dict) -> list[str]:
         f"actor_rollout_ref.model.path={_hydra_val(config['model_path'])}",
         "actor_rollout_ref.model.trust_remote_code=true",
         # packing the micro-batch into one row is only safe for a gdn hybrid when the child can
-        # reset linear-attention state at the example boundaries; see the caller's gdn gate.
-        "actor_rollout_ref.model.use_remove_padding="
-        + _hydra_val(config.get("use_remove_padding", True)),
+        # reset linear-attention state at the example boundaries -- and the caller's gdn gate now
+        # RAISES when it cannot, because the padded alternative cannot complete a step alongside the
+        # fused kernels below. always true by the time we get here.
+        "actor_rollout_ref.model.use_remove_padding=true",
         # 32k contexts: the fused linear-CE forward computes per-token log_probs from hidden states
         # + lm_head in chunks (FusedLinearForPPO), never materializing the [tokens, vocab] logits
         # tensor (~130 GB at 32k on a 248k vocab). the distillation loss consumes exactly
@@ -2480,15 +2481,11 @@ def run_opd_train(spec=None) -> None:
 
     # a gdn hybrid may only pack when the CHILD can honor seq_idx + cu_seqlens; the no-fla fallbacks
     # accept both and discard them, which silently bleeds state across packed example boundaries.
-    gdn_reset_arch = gdn_reset_arch_from_caps(caps, gdn_module) if gdn_hybrid else None
-    gdn_boundary_resets = gdn_reset_arch is not None
-    use_remove_padding = not gdn_hybrid or gdn_boundary_resets
-    if gdn_hybrid and not gdn_boundary_resets:
-        print(
-            "[opd] gdn hybrid without child-side boundary resets: disabling remove-padding so "
-            "packed examples cannot contaminate each other (slower, correct)",
-            flush=True,
-        )
+    # raises when the child cannot -- opd reaches no_padding_2_padding through
+    # trainer/distillation/losses.py exactly as grpo reaches it through ray_trainer, so the padded
+    # fallback that used to handle this cannot complete a step either. see
+    # require_gdn_boundary_resets.
+    gdn_reset_arch = require_gdn_boundary_resets(caps, gdn_module)
 
     # sm86's vllm 0.19.1 graph capture degenerates, so only that arch runs the rollout eagerly. grpo
     # has resolved this since the trl driver; opd never did, and opd is the MORE exposed of the two
@@ -2581,7 +2578,6 @@ def run_opd_train(spec=None) -> None:
     bridge.start()
     try:
         config = {
-            "use_remove_padding": use_remove_padding,
             "train_files": [train_file],
             "val_files": [val_file],
             "train_batch_size": prompts_per_step,
@@ -2912,7 +2908,7 @@ def run_opd_train(spec=None) -> None:
                 # whether it packed with resets or took the padded fallback. for a gate whose failure
                 # mode is silent contamination that is the one thing worth recording. None for a
                 # non-gdn model, where the question does not arise.
-                "gdn_boundary_resets": gdn_boundary_resets if gdn_hybrid else None,
+                "gdn_boundary_resets": gdn_hybrid or None,
                 "peak_gpu_gb": peak_gpu_gb,
                 "warm_started": bool(warmstart_adapter),
                 "resumed": bool(resume_step),

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/flash/engine/worker/perf/lifecycle.py

@@ -10,6 +10,14 @@ _CUDA_OOM_PREFLIGHT_RE = re.compile(
     r"free memory on device\s+cuda:\d+.*less than desired gpu memory utilization"
 )
 _CUDA_OOM_CACHE_BLOCKS = "no available memory for the cache blocks"
+# torch's own allocator OOM, which the two vllm STARTUP signatures above do not cover. the parent
+# classifies an in-process OOM off `torch.cuda.OutOfMemoryError` and the allocator counter, but a
+# verl child is a separate process: neither the exception type nor its `num_ooms` crosses, and the
+# terminal raise carries only "subprocess exited with status N". without this the one OOM shape that
+# happens DURING training reads as a permanent job_failed, so the lifecycle never retries it on a
+# larger gpu. anchored on torch's exact wording rather than a bare "out of memory", which a host-ram
+# OOM or an env's own error text would also match -- and those must not escalate the gpu.
+_CUDA_OOM_TORCH_RE = re.compile(r"(?:torch\.)?(?:cuda\.)?outofmemoryerror|cuda out of memory")
 
 
 class RetriableInfraError(RuntimeError):
@@ -42,6 +50,9 @@ def cuda_oom_message_evidence(message: str) -> str | None:
         return preflight.group(0)
     if _CUDA_OOM_CACHE_BLOCKS in normalized:
         return _CUDA_OOM_CACHE_BLOCKS
+    torch_oom = _CUDA_OOM_TORCH_RE.search(normalized)
+    if torch_oom is not None:
+        return torch_oom.group(0)
     return None
 
 

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/flash/engine/worker/rl_train.py

@@ -60,7 +60,6 @@ from flash.engine.worker.backend_common import (
     clamp_engine_len,
     export_peft_adapter,
     gdn_probe_module,
-    gdn_reset_arch_from_caps,
     kill_process_group,
     latest_global_step_dir,
     model_max_position_embeddings,
@@ -74,6 +73,7 @@ from flash.engine.worker.backend_common import (
     render_gdn_varlen_shim,
     render_tf32_shim,
     render_wandb_link_shim,
+    require_gdn_boundary_resets,
     resolve_blackwell_attention_backends,
     resolve_rollout_enforce_eager,
     resolve_verl_loggers,
@@ -387,8 +387,10 @@ def build_verl_overrides(cfg: dict) -> list[str]:
         # chunks (FusedLinearForPPO), never materializing the [tokens, vocab] logits tensor
         # (~130 GB at 32k on a 248k vocab). torch backend = numerically exact, no extra deps.
         # packing the micro-batch into one row is only boundary-safe for a gdn hybrid when the child
-        # can honor seq_idx + cu_seqlens; see the gdn gate in run_rl_train.
-        f"actor_rollout_ref.model.use_remove_padding={cfg.get('use_remove_padding', True)}",
+        # can honor seq_idx + cu_seqlens -- and run_rl_train now RAISES when it cannot, because the
+        # padded alternative cannot complete a step alongside the fused kernels below. so this is
+        # always true by the time we get here; there is no longer a configuration that turns it off.
+        "actor_rollout_ref.model.use_remove_padding=True",
         "actor_rollout_ref.model.use_fused_kernels=True",
         "actor_rollout_ref.model.fused_kernel_options.impl_backend=torch",
         *(
@@ -665,12 +667,10 @@ def _build_verl_training_cfg(
     experiment_name: str,
     gpu_type: str = "",
     n_gpus: int = 1,
-    use_remove_padding: bool = True,
 ) -> dict:
     engine_len = int(inp["engine_len"])
     sleep_unsupported = rollout_sleep_unsupported(inp["model_id"])
     return {
-        "use_remove_padding": use_remove_padding,
         "train_files": train_files,
         "val_files": val_files,
         "model_id": model_id,
@@ -3470,18 +3470,12 @@ def run_rl_train():
         # the shim is appended here rather than with the shims above because the answer needs
         # python_bin, resolved in the block above; sitecustomize is imported by the child, which has
         # not started yet.
-        gdn_reset_arch = gdn_reset_arch_from_caps(caps, gdn_module) if gdn_hybrid else None
-        gdn_boundary_resets = gdn_reset_arch is not None
-        use_remove_padding = not gdn_hybrid or gdn_boundary_resets
+        # raises when a gdn child cannot honor resets: the padded fallback that used to handle that
+        # case cannot complete a step on verl's fsdp engine. see require_gdn_boundary_resets.
+        gdn_reset_arch = require_gdn_boundary_resets(caps, gdn_module)
         if gdn_reset_arch is not None:
             with open(shim_py, "a") as f:
                 f.write(render_gdn_varlen_shim(gdn_reset_arch))
-        elif gdn_hybrid:
-            print(
-                "[grpo] gdn hybrid without child-side boundary resets: disabling remove-padding so "
-                "packed examples cannot contaminate each other (slower, correct)",
-                flush=True,
-            )
 
         expected_steps = int(inp["steps"])
         # verl logs from its own interpreter; gate wandb on that env (see resolve_verl_loggers).
@@ -3500,9 +3494,7 @@ def run_rl_train():
             )
         except Exception:  # no cuda / probe failure -> conservative bf16 kv
             _cc_ok = False
-        # reuse the gdn answer resolved above rather than re-probing: model_is_gdn_hybrid returns
-        # False when its own probe raises, so a second call can disagree with the first and turn
-        # fp8 kv ON for the very hybrid the comment above says it crashes.
+        # reuse the gdn answer resolved above rather than re-probing; see gdn_hybrid.
         fp8_kv = _cc_ok and not gdn_hybrid
         # one capability probe, both rollout decisions below. asked of the verl interpreter, whose
         # torch/vllm stack is the one that has to run the rollout.
@@ -3528,7 +3520,6 @@ def run_rl_train():
             enforce_eager=enforce_eager,
             attention_backend=attention_backend,
             mm_encoder_attn_backend=mm_encoder_attn_backend,
-            use_remove_padding=use_remove_padding,
             reward_path=reward_py,
             local_dir=local_dir,
             project_name=project_name,
@@ -3865,6 +3856,6 @@ def run_rl_train():
             reward_profile=reward_profile,
             step_intervals=_step_intervals(step_line_times),
             reward_bridge_batching=not inp["multi_turn"],
-            gdn_boundary_resets=gdn_boundary_resets if gdn_hybrid else None,
+            gdn_boundary_resets=gdn_hybrid or None,
         ),
     )

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/flash/runner/lifecycle.py

@@ -558,13 +558,31 @@ def _projected_retry_class(
     )
 
 
-def _oom_escalated(candidates, oom_vram_floor: int):
+def _candidate_usable_vram_gb(candidate) -> float:
+    """Run-usable VRAM for a candidate, under the same fit model the allocator selected it with.
+
+    `combined_vram_gb` is documented as THE single fit model, and every sizing path calls it. The raw
+    `gpu_count * vram_gb` product is NOT interchangeable with it for a sharded shape: it ignores the
+    replicated-per-card floor and the shard efficiency, so it overstates a multi-card box. Comparing
+    the two measures is what let an OOM retry move SIDEWAYS or DOWNWARDS -- 2x80 raw-counts as 160 GB
+    against a 141 GB single card that just OOM'd, but the allocator models that pair as 130.4 GB
+    usable, i.e. smaller than the shape that already failed.
+    """
+    from flash.providers.base import combined_vram_gb
+
+    return combined_vram_gb(candidate.vram_gb, int(getattr(candidate, "gpu_count", 1) or 1))
+
+
+def _oom_escalated(candidates, oom_vram_floor: float):
     """Candidates strictly LARGER than the VRAM that just OOM'd. ``oom_vram_floor == 0`` (no prior OOM)
     leaves the list unchanged; otherwise an 80GB OOM leaves only the >80GB classes (a same-size retry
-    would just OOM again). EMPTY means the run already OOM'd the largest available class."""
+    would just OOM again). EMPTY means the run already OOM'd the largest available class.
+
+    Both sides are measured with `_candidate_usable_vram_gb`, and the floor recorded on OOM uses it
+    too -- the filter is only meaningful if the floor and the candidates are on one scale."""
     if not oom_vram_floor:
         return list(candidates)
-    return [c for c in candidates if getattr(c, "gpu_count", 1) * c.vram_gb > oom_vram_floor]
+    return [c for c in candidates if _candidate_usable_vram_gb(c) > oom_vram_floor]
 
 
 def _await_runpod_completed_metrics(
@@ -743,7 +761,7 @@ def _submit_seed_supervised(
     # Grow only when an attempt actually provisioned a class and lost it to infra.
     failed_providers: set[str] = set()
     tried_classes: set[tuple[str, str, int]] = set()
-    oom_vram_floor = 0
+    oom_vram_floor = 0.0
     # Pin the environment ref ONCE, here, before any attempt runs -- not per attempt. Submit's pin
     # is best-effort, so a GitHub blip there leaves resolved_sha empty, and a managed slug points at
     # environment-hub@main, which moves. attempt_start > 0 means a previous invocation already ran an
@@ -906,9 +924,9 @@ def _submit_seed_supervised(
                     raise _cancel()
             cands = _oom_escalated(alloc.candidates, oom_vram_floor)
             if not cands:
-                last_detail = f"oom: exceeded the largest available GPU ({oom_vram_floor} GB)"
+                last_detail = f"oom: exceeded the largest available GPU ({oom_vram_floor:g} GB)"
                 print(
-                    f"seed={seed} OOM on the largest GPU class ({oom_vram_floor} GB); not retrying",
+                    f"seed={seed} OOM on the largest GPU class ({oom_vram_floor:g} GB); not retrying",
                     file=log,
                     flush=True,
                 )
@@ -1112,7 +1130,8 @@ def _submit_seed_supervised(
         last_detail = f"{res.failure}: {res.detail}"
         oom_shaped = res.failure == "oom"
         if oom_shaped and chosen is not None:
-            oom_vram_floor = max(oom_vram_floor, getattr(chosen, "gpu_count", 1) * chosen.vram_gb)
+            # same measure the filter compares against, see _candidate_usable_vram_gb
+            oom_vram_floor = max(oom_vram_floor, _candidate_usable_vram_gb(chosen))
         run_had_cache = bool(
             chosen is not None
             and getattr(get_provider(chosen.provider), "supports_weight_cache", False)
@@ -1166,7 +1185,7 @@ def _submit_seed_supervised(
         else:
             retry_target = "retrying (resume from last checkpoint)"
         action = (
-            f"retrying on a larger GPU (> {oom_vram_floor} GB)"
+            f"retrying on a larger GPU (> {oom_vram_floor:g} GB)"
             if (will_retry and oom_mode)
             else retry_target
             if will_retry

{freesolo_flash_dev-1.0.97 → freesolo_flash_dev-1.0.99}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.0.97"
+version = "1.0.99"
 description = "Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -121,7 +121,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.0.97"
+version = "1.0.99"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]