diffusers-workflow 0.4.0__py3-none-any.whl

dw/elision.py

@@ -0,0 +1,306 @@
+"""A step nothing references, and which saves nothing, does not run.
+
+`templates/minimax/dialogue-short` draws its two characters with Z-Image and
+references those portraits from every shot. An episode can just as well be
+cast from portraits that already exist - a shot entry's subject reference
+takes `from_file: "asset:cast/priya.jpg"` exactly as its voice references do -
+and then the two draw steps still ran and their output was discarded: roughly
+55 seconds and two model loads on portraits nothing in the run looked at
+(#109, #122). The recurring cast is the headline use of that template, so
+paying for it every episode was the wrong default, and no argument the caller
+could pass avoided it.
+
+So: before the first step executes, drop any step whose result no later step
+reads and which writes no file. It is static - it depends only on what the
+realized workflow references, never on a value produced during the run, which
+is what keeps it a different thing from conditional execution (#118 closed the
+step object specifically so an invented `when` is a hard error).
+
+Four guardrails, all of them load-bearing:
+
+- **A step that saves is kept.** `save` defaults to true, so a step declaring
+  a `result` is a deliverable unless it says otherwise - a workflow whose
+  whole point is writing three images references nothing.
+- **The last step is kept**, whatever it declares: it is the run's answer.
+- **A release moves rather than disappearing.** `release_pipeline` frees the
+  memory the step it sits on took; eliding it silently would leak that for
+  the rest of the run. It moves onto the last surviving step before it when
+  that step loaded the same pipeline, and is dropped when nothing was loaded
+  to release.
+- **Every elision is a warning.** Without one a misspelled reference makes
+  the step feeding it vanish and the failure moves from "previous result not
+  found" to "the picture is wrong". The static reference check refuses an
+  unresolvable literal reference, which contains most of it - the warning is
+  what covers the rest. The warning says which of the two happened: a step
+  the *written* definition reads through a variable the caller then
+  overrode was elided on purpose - it is `music-video`'s supplied singer
+  portrait, the saving the feature exists for - and telling that caller
+  their reference is probably misspelled puts a false positive on the
+  documented path, which is how the warnings channel stops being read
+  (#157). A step nothing ever read still gets the diagnosis.
+
+Elision is transitive: dropping a step can leave the step it read
+unreferenced in turn, so it runs to a fixed point.
+"""
+
+import logging
+
+from .step_cache import reference_resolves_to, referenced_result_names
+
+VARIABLE_PREFIX = "variable:"
+
+logger = logging.getLogger("dw")
+
+
+def _saves(step):
+    """Whether the step writes a file.
+
+    Exactly what `Result.save` asks: a `content_type` to write, and `save`
+    not turned off. `save` defaults to true, so declaring a `result` with a
+    content type is declaring a deliverable - and a step with no `result` at
+    all writes nothing, whatever it generates, which is what makes the
+    portrait steps droppable in the first place.
+
+    A composing step is the one exception: its child workflow saves on its
+    own terms, so the parent's `result` block (or its absence) says nothing
+    about whether files are written. It is kept.
+    """
+    if "workflow" in step:
+        return True
+    result = step.get("result")
+    if not isinstance(result, dict):
+        return False
+    if result.get("content_type") is None:
+        return False
+    return result.get("save", True) is not False
+
+
+def _reused_component_names(steps):
+    """Every component name a later step asks an earlier one to have shared.
+
+    Component sharing is keyed on the component's name rather than on the
+    step's, so the step that shares is referenced without ever being named.
+    """
+    names = set()
+    for step in steps:
+        pipeline = step.get("pipeline")
+        if isinstance(pipeline, dict):
+            for name in pipeline.get("reused_components") or []:
+                names.add(name)
+    return names
+
+
+def _referenced_pipeline_steps(steps):
+    """Every step name a later `pipeline_reference` addresses."""
+    names = set()
+    for step in steps:
+        reference = step.get("pipeline_reference")
+        if isinstance(reference, dict):
+            name = reference.get("reference_name")
+            if isinstance(name, str):
+                names.add(name)
+    return names
+
+
+def _needed_by(step, later):
+    """Whether any of `later` reads this step - by result, by pipeline, or by
+    a component it shares."""
+    name = step.get("name")
+    if not isinstance(name, str):
+        return True
+
+    if any(reference_resolves_to(ref, name) for ref in referenced_result_names(later)):
+        return True
+    if name in _referenced_pipeline_steps(later):
+        return True
+
+    pipeline = step.get("pipeline")
+    shared = set((pipeline or {}).get("shared_components") or [])
+    return bool(shared & _reused_component_names(later))
+
+
+def _carry_release(elided, kept):
+    """Move an elided step's release onto the last surviving step before it.
+
+    `release_pipeline` names the pipeline the elided step itself loaded, so
+    it is only meaningful on an earlier step that loaded the same one -
+    moving it anywhere else would unload something the workflow did not ask
+    to unload. `release_models` frees the process-wide task-model cache, so
+    the last step that ran before this one is exactly where it belongs. With
+    nothing before it, nothing was loaded and the flag is dropped.
+    """
+    from .workflow import pipeline_cache_key
+
+    if not kept:
+        return False
+    predecessor = kept[-1]
+    carried = False
+    if elided.get("release_models"):
+        predecessor["release_models"] = True
+        carried = True
+    if not elided.get("release_pipeline"):
+        return carried
+    elided_pipeline = elided.get("pipeline")
+    kept_pipeline = predecessor.get("pipeline")
+    if not elided_pipeline or not kept_pipeline:
+        return carried
+    if pipeline_cache_key(elided_pipeline) == pipeline_cache_key(kept_pipeline):
+        predecessor["release_pipeline"] = True
+        carried = True
+    return carried
+
+
+def overriding_variables(written, substituted_steps):
+    """{step name: variable name} for every step the workflow as *written*
+    reads only through a variable whose value no longer names it.
+
+    `music-video` declares `singer_reference` as
+    `{"from_previous_result": "draw_singer"}` and every shot references the
+    variable; a caller who passes a portrait instead replaces that value, and
+    `draw_singer` becomes unreferenced *because they said so*. Comparing the
+    two definitions is what tells that apart from a misspelling, and it costs
+    one walk of the variables block (#157).
+
+    `written` is the definition as the author wrote it - before substitution,
+    so the defaults are still there; `substituted_steps` is the expanded step
+    list elision is about to judge.
+    """
+    variables = (written or {}).get("variables")
+    if not isinstance(variables, dict):
+        return {}
+    written_steps = (written or {}).get("steps")
+    if not isinstance(written_steps, list):
+        return {}
+    still_read = referenced_result_names(substituted_steps)
+    overridden = {}
+    for variable, value in variables.items():
+        # A variable no step reads is not how the step was reached, so its
+        # disappearance is nobody's decision - the general diagnosis is
+        # right for that one
+        if not _reads_variable(written_steps, variable):
+            continue
+        for name in referenced_result_names([value]):
+            if any(reference_resolves_to(read, name) for read in still_read):
+                continue
+            overridden.setdefault(name, variable)
+    return overridden
+
+
+def _reads_variable(tree, name):
+    """Whether anything in `tree` references 'variable:<name>'."""
+    reference = VARIABLE_PREFIX + name
+    if isinstance(tree, str):
+        return tree == reference
+    if isinstance(tree, dict):
+        return any(_reads_variable(value, name) for value in tree.values())
+    if isinstance(tree, list):
+        return any(_reads_variable(value, name) for value in tree)
+    return False
+
+
+def elide_unreferenced_steps(steps, overridden=None):
+    """The steps that will run, and what was dropped.
+
+    Returns (kept, elided) where `elided` is
+    [{'step': name, 'reason': str, 'overridden_by': variable | absent}],
+    in the order the steps were written. `overridden` is
+    `overriding_variables`' answer: a step named there was dropped because a
+    caller replaced the reference that read it, which is a different event
+    from a step nothing reads by accident, and is reported as one. `steps` is the expanded, substituted
+    list - a `for_each` member is a step like any other by then, and `gather:`
+    has already become the `previous_result:` list it stands for.
+
+    The list is not copied: a kept step is the same object that went in, and
+    a release carried onto a survivor is written into it.
+    """
+    if not isinstance(steps, list) or len(steps) < 2:
+        return steps, []
+
+    kept = list(steps)
+    elided = []
+    changed = True
+    while changed:
+        changed = False
+        for index, step in enumerate(kept):
+            if index == len(kept) - 1:
+                # The run's answer, whatever it declares
+                continue
+            if not isinstance(step, dict) or _saves(step):
+                continue
+            if _needed_by(step, kept[index + 1 :]):
+                continue
+            variable = (overridden or {}).get(step.get("name"))
+            reason = (
+                f"'{variable}' was supplied, so nothing reads its result"
+                if variable
+                else "nothing after it reads its result and it saves no file"
+            )
+            if _carry_release(step, kept[:index]):
+                reason += "; its release was carried onto the step before it"
+            entry = {"step": step.get("name"), "reason": reason}
+            if variable:
+                entry["overridden_by"] = variable
+            elided.append(entry)
+            del kept[index]
+            changed = True
+            break
+
+    # Written order, not the order the fixed point happened to reach them in
+    order = {
+        step.get("name"): index
+        for index, step in enumerate(steps)
+        if isinstance(step, dict)
+    }
+    elided.sort(key=lambda entry: order.get(entry["step"], 0))
+    return kept, elided
+
+
+def elide_definition(workflow_def, written=None):
+    """elide_unreferenced_steps over a whole definition, in place.
+
+    Returns the elided-step records; the definition's `steps` is replaced
+    when anything was dropped, so a caller that wants the untouched list must
+    keep its own copy. `written` is the same workflow before substitution,
+    which is what distinguishes a step a caller deliberately replaced from
+    one nothing reads by accident (#157); without it every elision reads as
+    the second.
+    """
+    steps = workflow_def.get("steps")
+    overridden = (
+        overriding_variables(written, steps if isinstance(steps, list) else [])
+        if written
+        else None
+    )
+    kept, elided = elide_unreferenced_steps(steps, overridden)
+    if elided:
+        workflow_def["steps"] = kept
+    return elided
+
+
+def warn_elided(elided):
+    """Say what did not run, where whoever asked for the run can read it.
+
+    A warning rather than a log line for the reason every run-time warning is
+    one: a consumer over the API or MCP sees the job's `warnings` list and
+    nothing else (#82), and "the portrait step was skipped" is precisely the
+    thing that explains an otherwise inexplicable result.
+    """
+    from .events import emit_warning
+
+    for entry in elided:
+        # A step whose reference the caller replaced was elided on purpose -
+        # say what was saved, not what might be wrong with it (#157)
+        advice = (
+            ""
+            if entry.get("overridden_by")
+            else (
+                " If it was meant to, either a later step's reference to it "
+                "is misspelled or it needs a 'result' to save."
+            )
+        )
+        emit_warning(
+            f"Step '{entry['step']}' did not run: {entry['reason']}.{advice}",
+            kind="step_elided",
+            step=entry["step"],
+            overridden_by=entry.get("overridden_by"),
+        )

dw/events.py

@@ -0,0 +1,275 @@
+"""Progress reporting and cooperative cancellation for workflow runs.
+
+A RunContext travels ambiently (contextvars) through a run: Workflow.run
+activates it, and Step/Pipeline reach it with get_context() rather than
+threading a parameter through every action signature. Callers that never
+pass a context get a no-op one, so the CLI path pays nothing.
+"""
+
+import logging
+import threading
+import time
+import contextvars
+
+logger = logging.getLogger("dw")
+
+# How long a phase may go without any event before the watchdog speaks up,
+# and (since emitting a stall event itself counts as an event - see
+# RunContext._watchdog_loop) the cadence it repeats on while the stall
+# continues. 30s is well past the 7-10s lead-in a healthy step-callback
+# pipeline shows before its first `pipeline_step` (see
+# docs/proposals/step-callback-lead-in-instrumentation.md) but short enough
+# that a genuine stall is visible long before a human would give up on it.
+# The report goes to the event log only - the job's persisted `warnings`
+# list filters `kind == "phase_stall"` (dw/server/jobs.py), because a stall
+# that resolved is not a warning about the result.
+PHASE_STALL_THRESHOLD_SECONDS = 30.0
+# How often the watchdog thread wakes to check - independent of the
+# threshold above, just fine-grained enough that the reported
+# seconds_since_phase_start doesn't overshoot by much.
+PHASE_STALL_CHECK_INTERVAL_SECONDS = 5.0
+
+
+class WorkflowCancelled(Exception):
+    """Raised inside a run when its RunContext has been cancelled."""
+
+
+class RunContext:
+    """Carries the event sink and the cancellation flag for one workflow run.
+
+    cancel() may be called from any thread (the worker's command watcher);
+    everything else runs on the thread executing the workflow. The
+    phase-stall watchdog (_watchdog_loop) is the same story - it reads
+    _current_phase/_last_event_at from its own background thread with no
+    lock, which is fine for simple attribute reads/writes under the GIL.
+    """
+
+    def __init__(self, on_event=None):
+        self._on_event = on_event
+        self._cancel = threading.Event()
+        # The most recent phase this run reported (see emit_phase/PHASES) -
+        # cancel() reads it to tell a client whether the cancel takes effect
+        # right away or has to wait out an in-flight, non-interruptible phase
+        self._current_phase = None
+        # Pipeline cache keys this run resolved - the worker evicts entries a
+        # run no longer touches, so an edited workflow drops stale models
+        self.touched_pipelines = set()
+        # Watchdog state. monotonic, not wall clock, since a stall is
+        # measured in elapsed time, not affected by clock adjustments.
+        self._last_event_at = time.monotonic()
+        self._phase_started_at = time.monotonic()
+        # Last *real* event - anything but the watchdog's own phase_stall
+        # report - and what kind it was. Tracked separately from
+        # _last_event_at (which the stall report itself also bumps, to drive
+        # its own repeat cadence) so "how long has it actually been quiet"
+        # doesn't reset every time the watchdog speaks.
+        self._last_progress_at = self._last_event_at
+        self._last_progress_kind = None
+        # Reference-counted: a sub-workflow runs inside its parent's
+        # RunContext (Workflow.run reuses the ambient one), so the watchdog
+        # starts on the outermost run() and stops on the outermost's exit,
+        # not on every nested one
+        self._run_depth = 0
+        self._watchdog_thread = None
+        self._watchdog_stop = threading.Event()
+
+    def emit(self, event_type, counts_as_progress=True, **data):
+        """Send one event to the sink. counts_as_progress=False is for a
+        status report that says nothing has moved - a download reporting
+        zero new bytes (#343): the caller should see it, but the stall
+        watchdog must still see the silence, so it bumps neither clock."""
+        now = time.monotonic()
+        if counts_as_progress:
+            self._last_event_at = now
+            if not (event_type == "warning" and data.get("kind") == "phase_stall"):
+                self._last_progress_at = now
+                self._last_progress_kind = event_type
+        if self._on_event is None:
+            return
+        try:
+            self._on_event({"event": event_type, **data})
+        except Exception as e:
+            # A broken sink must not kill a run that is otherwise fine
+            logger.warning(f"Progress event sink failed on '{event_type}': {e}")
+
+    def note_phase(self, phase):
+        """Record the run's latest phase - called by emit_phase, read by
+        cancel() to judge whether the current phase can be interrupted, and
+        by the watchdog to time how long the current phase has run."""
+        self._current_phase = phase
+        self._phase_started_at = time.monotonic()
+
+    def enter_run(self):
+        """Called around Workflow.run - starts the watchdog on the
+        outermost call, a no-op on a nested (sub-workflow) one."""
+        self._run_depth += 1
+        if self._run_depth == 1:
+            self._watchdog_stop.clear()
+            self._watchdog_thread = threading.Thread(
+                target=self._watchdog_loop, daemon=True, name="dw-phase-stall"
+            )
+            self._watchdog_thread.start()
+
+    def exit_run(self):
+        """Matching exit for enter_run - stops the watchdog once the
+        outermost run() has left, whether it succeeded or raised."""
+        self._run_depth -= 1
+        if self._run_depth == 0 and self._watchdog_thread is not None:
+            self._watchdog_stop.set()
+            self._watchdog_thread.join(timeout=PHASE_STALL_CHECK_INTERVAL_SECONDS * 2)
+            self._watchdog_thread = None
+
+    def _watchdog_loop(self):
+        """Generic, pipeline-agnostic: this only ever looks at 'how long
+        since any event' and 'what phase are we in', never at what a
+        particular pipeline does inside a phase (see
+        docs/proposals/step-callback-lead-in-instrumentation.md, Option A).
+        A stall report is itself an event, so it bumps _last_event_at and
+        naturally repeats on PHASE_STALL_THRESHOLD_SECONDS while the silence
+        continues and stops the moment a real progress event arrives -
+        but it deliberately does not bump _last_progress_at, so the
+        seconds_since_last_progress it reports keeps climbing across
+        repeats rather than resetting itself every 30s (#357). It is a
+        watchdog notice, not progress: a caller polling events should not
+        read a run of these, or a climbing event_count made of them, as
+        liveness.
+        """
+        while not self._watchdog_stop.wait(PHASE_STALL_CHECK_INTERVAL_SECONDS):
+            phase = self._current_phase
+            if phase is None:
+                continue
+            now = time.monotonic()
+            if now - self._last_event_at < PHASE_STALL_THRESHOLD_SECONDS:
+                continue
+            seconds_since_phase_start = round(now - self._phase_started_at, 1)
+            seconds_since_last_progress = round(now - self._last_progress_at, 1)
+            last_kind = self._last_progress_kind or "phase start"
+            last_offset = round(
+                max(0.0, self._last_progress_at - self._phase_started_at), 1
+            )
+            message = (
+                f"no progress event for {seconds_since_last_progress:.1f}s in phase "
+                f"'{phase}' (last: {last_kind} at +{last_offset:.1f}s) - informational; "
+                "long silent stretches are normal for some models, see the model's skill"
+            )
+            logger.warning(message)
+            self.emit(
+                "warning",
+                message=message,
+                kind="phase_stall",
+                phase=phase,
+                seconds_since_phase_start=seconds_since_phase_start,
+                seconds_since_last_progress=seconds_since_last_progress,
+            )
+
+    def cancel(self):
+        self._cancel.set()
+        if self._current_phase in NON_INTERRUPTIBLE_PHASES:
+            # A model load or a task step has no checkpoint to catch this
+            # flag until it finishes - say so, or the client sees the cancel
+            # request go silent for however long that takes
+            self.emit("cancel_pending", phase=self._current_phase)
+
+    @property
+    def cancelled(self):
+        return self._cancel.is_set()
+
+    def check_cancelled(self):
+        if self._cancel.is_set():
+            raise WorkflowCancelled("Workflow run was cancelled")
+
+    def touch_pipeline(self, cache_key):
+        self.touched_pipelines.add(cache_key)
+
+
+_active_context = contextvars.ContextVar("dw_run_context", default=None)
+
+
+def get_context():
+    """The active run's context, or a no-op one outside any run."""
+    context = _active_context.get()
+    return context if context is not None else RunContext()
+
+
+def current_context():
+    """The active run's context, or None outside any run."""
+    return _active_context.get()
+
+
+def activate_context(context):
+    """Make a context the active one; returns a token for deactivate_context."""
+    return _active_context.set(context)
+
+
+def deactivate_context(token):
+    _active_context.reset(token)
+
+
+# The coarse states a run passes through. Small on purpose: a phase says what
+# the run is waiting on, not what any one library is doing internally
+PHASES = ("loading", "cached", "generating", "decoding", "saving", "task")
+
+# Phases with no checkpoint of their own: nothing inside a from_pretrained()
+# call or a task handler consults the cancel flag, so a cancel requested
+# during one of these can only take effect once the phase finishes on its own
+NON_INTERRUPTIBLE_PHASES = ("loading", "task")
+
+
+def select_kinds(events, kinds):
+    """The events whose `event` or `kind` is one of `kinds`, in order.
+
+    Both, because a consumer names what it sees: the bookkeeping events are
+    told apart by `event` (`log`, `warning`, `memory`), but a warning's
+    own type - `phase_stall`, `audio_clipped` - is its `kind`, and matching
+    `event` alone quietly returned an empty page for those. No `kinds`
+    means no filter."""
+    if not kinds:
+        return list(events)
+    allowed = set(kinds)
+    return [e for e in events if e.get("event") in allowed or e.get("kind") in allowed]
+
+
+def emit_warning(message, **data):
+    """Report something the run's result carries but its status will not.
+
+    A warning a step discovers at run time - shots being cut together 10 dB
+    apart, a video about to be written at a frame rate nothing chose - is
+    only useful where whoever asked for the run can read it. The server's
+    own log is not that place: a consumer over the API or MCP sees the event
+    stream and the job's `warnings` list and nothing else, so a diagnostic
+    that only reaches the log does not exist out there (#82).
+
+    Logged as well as emitted, because the CLI and the REPL have no event
+    sink and the log is the whole of their surface.
+    """
+    logger.warning(message)
+    get_context().emit("warning", message=message, **data)
+
+
+def emit_phase(phase, detail=None):
+    """Report a coarse phase change on the active run.
+
+    A step spends most of its wall clock outside the denoise loop - pulling
+    weights, decoding latents, encoding video - and a step counter says
+    nothing about any of that. These are the rest of the story. They are rare
+    enough (a handful per step) to carry a free-text detail alongside, which
+    is what makes 'loading' readable as 'which model'.
+    """
+    context = get_context()
+    context.note_phase(phase)
+    context.emit("phase", phase=phase, detail=detail)
+
+
+def emit_log(message, **data):
+    """Narrate one step of a long, otherwise silent stretch of a run.
+
+    A `log` event rather than a phase: `PHASES` is a closed set a consumer
+    switches on, and "which file is being written" is a detail inside one of
+    them, not a new state. The modular block lead-in (#95) is the same shape
+    at the other end of a step.
+
+    Logged as well as emitted, because the CLI and the REPL have no event
+    sink and the log is the whole of their surface.
+    """
+    logger.info(message)
+    get_context().emit("log", message=message, **data)