diffusers-workflow 0.4.0__py3-none-any.whl

dw/server/assess.py

@@ -0,0 +1,132 @@
+"""Assess a finished output in the server process (#388).
+
+`GET /api/gallery/{name}/assess` runs the assessment probes
+(`dw/tasks/assess.py`) against one gallery file or asset without queueing
+anything: the probes are CPU-only and read a file, so the route is a sync
+`def` FastAPI runs in its threadpool, beside whatever job holds the GPU.
+
+One request is one decode: the file is streamed once into a `Media` and
+every applicable probe reads that. The shot boundaries are the ones the
+caller's root recorded - the run manifest for an output, the sidecar
+`keep_output` wrote for an asset (#393) - resolved here rather than by the
+probe, which cannot know which workspace the file belongs to.
+
+What comes back are places to look, not verdicts; nothing acts on a
+finding (`dw/assessment_rules.py`).
+"""
+
+import json
+
+from ..tasks.assess import (
+    read_media,
+    seams_answer,
+    shots_answer,
+    sync_drift_answer,
+)
+
+PROBES = {
+    "analyze_shots": shots_answer,
+    "analyze_seams": seams_answer,
+    "analyze_sync_drift": sync_drift_answer,
+}
+
+
+def unknown_probe(probe):
+    """The 400 detail for a probe outside the whitelist, or None."""
+    if probe is None or probe in PROBES:
+        return None
+    return f"Unknown probe {probe!r} - one of {', '.join(PROBES)}"
+
+
+def _dedupe_findings(findings):
+    """Findings with exact duplicates dropped, order preserved.
+
+    `_shot_span_findings` (`dw/tasks/assess.py`) is shared by all three
+    probes and raises the identical finding from each one that ran, so a
+    file with a span overrun listed it three times in the compact merge
+    (#427) - one true finding, not three.
+    """
+    seen = set()
+    deduped = []
+    for found in findings:
+        key = json.dumps(found, sort_keys=True, default=str)
+        if key in seen:
+            continue
+        seen.add(key)
+        deduped.append(found)
+    return deduped
+
+
+def _not_applicable(kind, media, records):
+    """Why each probe cannot say anything about this file: {probe: why}."""
+    if media is None:
+        why = (
+            "a still has no shots, seams or soundtrack to measure"
+            if kind == "image"
+            else "not a video or audio file"
+        )
+        return dict.fromkeys(PROBES, why)
+    reasons = {}
+    if media.audio is None:
+        reasons["analyze_shots"] = "no audio track"
+        reasons["analyze_sync_drift"] = "no audio track"
+    elif media.thumbs is None:
+        reasons["analyze_sync_drift"] = "no picture to sync against"
+    if not records or len(records) < 2:
+        reasons["analyze_seams"] = (
+            "no shot boundaries recorded for this file - analyze_seams "
+            "needs two or more shots"
+        )
+    return reasons
+
+
+def assess(path, kind, shots, probe=None, detail=False):
+    """Run the applicable probes over one file.
+
+    path: the validated file; kind: its MEDIA_KINDS entry ("video", "audio",
+    "image", ...); shots: the shot records its root recorded, or None.
+
+    With `probe`, that probe's full answer. Otherwise every applicable
+    probe's findings merged, with `rules_applied`, `rules_skipped` and
+    `not_applicable`; `detail` adds each probe's full answer under `probes`.
+    """
+    media = read_media(path) if kind in ("audio", "video") else None
+    records = [dict(shot) for shot in shots] if shots else None
+    source = "manifest" if records else "none"
+    reasons = _not_applicable(kind, media, records)
+
+    if probe is not None:
+        # Asked by name: a probe the file cannot feed at all says why; one
+        # it can (a shotless seam pass included) answers in full, its own
+        # rules_skipped saying what could not be measured
+        blocked = reasons.get(probe)
+        if media is None or (blocked and probe != "analyze_seams"):
+            return {"probe": probe, "not_applicable": {probe: blocked}}
+        return {"probe": probe, **PROBES[probe](media, records, source)}
+
+    answers = {
+        name: run(media, records, source)
+        for name, run in PROBES.items()
+        if name not in reasons
+    }
+    body = {
+        "shots_source": source,
+        "findings": _dedupe_findings(
+            found for answer in answers.values() for found in answer["findings"]
+        ),
+        "rules_applied": [
+            rule for answer in answers.values() for rule in answer["rules_applied"]
+        ],
+        "rules_skipped": [
+            {"probe": name, **skipped}
+            for name, answer in answers.items()
+            for skipped in answer["rules_skipped"]
+        ],
+        "not_applicable": reasons,
+    }
+    if detail:
+        body["probes"] = answers
+    return body
+
+
+__all__ = ["PROBES", "assess", "unknown_probe"]

dw/server/catalog_shape.py

@@ -0,0 +1,487 @@
+"""What a workflow makes, read off its definition.
+
+A catalog entry is chosen by shape before anything else - one still, a set,
+one shot, a cut sequence - and the definition already says which, in the
+steps it has and what they feed each other. Deriving it here means the
+vocabulary cannot go stale against the file and needs no backfill; a
+declared `shape`/`traits`/`summary` overrides for the odd workflow the
+rules misread.
+
+Reads the raw JSON only: no variable substitution, no type loading, so it
+costs a dict walk and behaves the same on a repo template and a file an
+agent saved a second ago. Nothing here names a model family - the rules
+read structure (a concat step, a `references` argument), never checkpoints.
+"""
+
+import re
+
+from ..for_each import list_fields
+from ..variable_constraints import declared_constraints, entry_constraint_fields
+
+SHAPES = (
+    "image",
+    "image-set",
+    "image-edit",
+    "shot",
+    "sequence",
+    "audio",
+    "text",
+    "utility",
+)
+TRAITS = (
+    "has-audio",
+    "chained",
+    "image-conditioned",
+    "identity-referenced",
+    "needs-input-media",
+    "composes-workflows",
+)
+# Tasks that create content rather than process it. A workflow made only
+# of processing tasks is a utility.
+GENERATIVE_TASKS = frozenset(
+    {
+        "generate_speech",
+        "text_generation",
+        "image_to_text",
+        "diffusion_upscale",
+        "interpolate_frames",
+    }
+)
+SUMMARY_LIMIT = 120
+
+_KIND_PRECEDENCE = ("video", "audio", "image", "text")
+_EDIT_PIPELINE = re.compile(r"inpaint|img2img|edit|upscale|outpaint", re.I)
+_CHAIN_ARGUMENTS = frozenset(
+    {"last_frame", "last_segment", "last_image", "match_audio"}
+)
+_MEDIA_ARGUMENTS = frozenset(
+    {"image", "video", "audio", "mask_image", "urls", "videos", "clip"}
+)
+_CUT_TASKS = frozenset({"concat_videos", "dissolve_videos"})
+# Components that exist only to synthesise a waveform. A video pipeline
+# carrying one emits an audio track whether or not it says so in `output`.
+_AUDIO_COMPONENTS = frozenset({"vocoder", "audio_vae"})
+_SENTENCE_END = re.compile(r"(?<=[.!?])\s|\n")
+
+
+def _steps(definition):
+    steps = definition.get("steps") if isinstance(definition, dict) else None
+    return (
+        [step for step in steps if isinstance(step, dict)]
+        if isinstance(steps, list)
+        else []
+    )
+
+
+def _block(step):
+    """The step's one body: pipeline, pipeline_reference, task or workflow."""
+    for key in ("pipeline", "pipeline_reference", "task", "workflow"):
+        body = step.get(key)
+        if isinstance(body, dict):
+            return key, body
+    return None, {}
+
+
+def _arguments(step):
+    _kind, body = _block(step)
+    arguments = body.get("arguments")
+    return arguments if isinstance(arguments, dict) else {}
+
+
+def _kind(step):
+    result = step.get("result")
+    if isinstance(result, dict) and isinstance(result.get("content_type"), str):
+        return result["content_type"].split("/")[0]
+    return None
+
+
+def _generates(step):
+    """Whether the step creates content: a pipeline, a reference to one, a
+    sub-workflow, or a generative task."""
+    key, body = _block(step)
+    if key in ("pipeline", "pipeline_reference", "workflow"):
+        return True
+    return key == "task" and body.get("command") in GENERATIVE_TASKS
+
+
+def _component_type(step):
+    key, body = _block(step)
+    if key != "pipeline":
+        return ""
+    configuration = body.get("configuration")
+    if not isinstance(configuration, dict):
+        return ""
+    return str(configuration.get("component_type", ""))
+
+
+def _fed_by(value):
+    """Distinct step names a value's previous_result references name."""
+    found = set()
+    if isinstance(value, str) and value.startswith("previous_result:"):
+        found.add(value.split(":", 1)[1].split(".", 1)[0])
+    elif isinstance(value, list):
+        for item in value:
+            found |= _fed_by(item)
+    elif isinstance(value, dict):
+        for item in value.values():
+            found |= _fed_by(item)
+    return found
+
+
+def _walk(value):
+    yield value
+    if isinstance(value, dict):
+        for item in value.values():
+            yield from _walk(item)
+    elif isinstance(value, list):
+        for item in value:
+            yield from _walk(item)
+
+
+def _needs_input_media(steps):
+    for step in steps:
+        arguments = _arguments(step)
+        for name, value in arguments.items():
+            if name not in _MEDIA_ARGUMENTS:
+                continue
+            # A list argument (gather_images' `urls`) carries the same fact
+            # one level in.
+            candidates = value if isinstance(value, list) else [value]
+            if any(
+                isinstance(item, str) and item.startswith("variable:")
+                for item in candidates
+            ):
+                return True
+        for value in _walk(arguments):
+            if isinstance(value, str) and value.startswith("asset:"):
+                return True
+            if isinstance(value, dict) and "location" in value:
+                return True
+    return False
+
+
+def _cuts_together(steps):
+    """A concat or dissolve fed by two or more distinct steps, or by a list
+    of shots handed in whole - one `variable:` reference is a supplied list
+    whose length only the caller knows, one `gather:` reference is every
+    member of a for_each group, and a cut over either is still an edit."""
+    for step in steps:
+        key, body = _block(step)
+        if key == "task" and body.get("command") in _CUT_TASKS:
+            videos = _arguments(step).get("videos")
+            if isinstance(videos, str) and videos.startswith(("variable:", "gather:")):
+                return True
+            sources = _fed_by(videos)
+            if isinstance(videos, list):
+                # Each supplied shot - a variable, an asset, a path - is its
+                # own source; two previous_result entries naming one step are not
+                sources |= {
+                    item
+                    for item in videos
+                    if isinstance(item, str) and not item.startswith("previous_result:")
+                }
+            if len(sources) >= 2:
+                return True
+    return False
+
+
+def _audio_components(step):
+    """Audio-only components the step's pipeline configures."""
+    key, body = _block(step)
+    if key != "pipeline":
+        return set()
+    configuration = body.get("configuration")
+    if not isinstance(configuration, dict) or not isinstance(
+        configuration.get("components"), dict
+    ):
+        return set()
+    return set(configuration["components"]) & _AUDIO_COMPONENTS
+
+
+def _derive_shape(steps, kind):
+    # Cutting shots together makes a sequence even when every shot was
+    # supplied rather than generated - an edit is what comes out, and the
+    # utility fallthrough would otherwise hide the assembly templates.
+    if kind == "video" and _cuts_together(steps):
+        return "sequence"
+    if kind is None or not any(_generates(step) for step in steps):
+        return "utility"
+    if kind == "text":
+        return "text"
+    if kind == "audio":
+        return "audio"
+    if kind == "video":
+        return "shot"
+    image_steps = [
+        step for step in steps if _generates(step) and _kind(step) == "image"
+    ]
+    for step in image_steps:
+        if _EDIT_PIPELINE.search(_component_type(step)):
+            return "image-edit"
+        if {"image", "mask_image"} & set(_arguments(step)):
+            return "image-edit"
+    if len(image_steps) >= 2 or any(
+        _block(step)[0] == "workflow" for step in image_steps
+    ):
+        return "image-set"
+    return "image"
+
+
+def _derive_traits(steps):
+    """The independent facts about how the output is made or what it needs.
+
+    `has-audio` says the workflow emits a generated audio track - a step
+    whose own result is a waveform, a speech task, a video pipeline asked
+    for audio, or one carrying a component that exists only to synthesise
+    one. Not specifically dialogue, and not a track the workflow was handed.
+    """
+    traits = set()
+    for step in steps:
+        key, body = _block(step)
+        arguments = _arguments(step)
+        if key == "task" and body.get("command") == "generate_speech":
+            traits.add("has-audio")
+        # A pipeline whose own result is a waveform generates one. The three
+        # checks below it all ask about a *video* step that also carries
+        # audio, which left Music 3's template - the one entry in the catalog
+        # whose whole output is a track - answering a `has-audio` filter with
+        # nothing.
+        if _generates(step) and _kind(step) == "audio":
+            traits.add("has-audio")
+        output = arguments.get("output")
+        if _kind(step) == "video" and isinstance(output, list) and "audio" in output:
+            traits.add("has-audio")
+        if _kind(step) == "video" and _audio_components(step):
+            traits.add("has-audio")
+        if key == "pipeline" and "chain" in body:
+            traits.add("chained")
+        if _CHAIN_ARGUMENTS & set(arguments):
+            traits.add("chained")
+        if _kind(step) == "video" and (
+            "image" in arguments or "ImageToVideo" in _component_type(step)
+        ):
+            traits.add("image-conditioned")
+        if "references" in arguments:
+            traits.add("identity-referenced")
+        if key == "workflow":
+            traits.add("composes-workflows")
+    if _needs_input_media(steps):
+        traits.add("needs-input-media")
+    return sorted(traits)
+
+
+def _truncate(text):
+    """One line, cut at a word boundary and elided if it runs past the limit.
+
+    A declared summary goes through this too: `workflow_details` reads a file
+    without validating it, so the schema's maxLength never runs on that path
+    and a long declaration would otherwise reach a listing unclipped.
+    """
+    if len(text) <= SUMMARY_LIMIT:
+        return text, False
+    cut = text[: SUMMARY_LIMIT - 1]
+    cut = cut[: cut.rfind(" ")] if " " in cut else cut
+    return cut.rstrip() + "…", True
+
+
+def _derive_summary(description):
+    text = str(description or "").strip()
+    if not text:
+        return "", False
+    return _truncate(_SENTENCE_END.split(text, maxsplit=1)[0].strip())
+
+
+def derive_catalog_metadata(definition):
+    """shape, traits and summary for one definition, declarations honoured.
+
+    Returns {shape, traits, summary, summary_truncated, lists, declared};
+    `lists` names what an entry of each for_each-driven variable has to
+    carry (`list_fields`) plus its default's length, so an agent can write
+    the list argument without opening the definition; `declared` names
+    which of the three prose fields came from the file rather than the
+    rules, so a test can refuse a declaration that merely repeats the
+    derivation.
+    """
+    if not isinstance(definition, dict):
+        definition = {}
+    steps = _steps(definition)
+    kinds = {_kind(step) for step in steps} - {None}
+    kind = next((k for k in _KIND_PRECEDENCE if k in kinds), None)
+
+    declared = set()
+    shape = _derive_shape(steps, kind)
+    if definition.get("shape") in SHAPES:
+        shape = definition["shape"]
+        declared.add("shape")
+    traits = _derive_traits(steps)
+    if isinstance(definition.get("traits"), list):
+        traits = sorted(t for t in definition["traits"] if t in TRAITS)
+        declared.add("traits")
+    summary, truncated = _derive_summary(definition.get("description"))
+    if isinstance(definition.get("summary"), str) and definition["summary"].strip():
+        summary, truncated = _truncate(definition["summary"].strip())
+        declared.add("summary")
+    variables = definition.get("variables")
+    variables = variables if isinstance(variables, dict) else {}
+    # A bound an entry field carries is reported beside that field, not
+    # only in the top-level `constraints` block: a caller reading what a
+    # `shots` entry takes reads the rule for `num_frames` there (#145)
+    rules = declared_constraints(definition)
+    entry_rules = entry_constraint_fields(definition)
+    lists = {
+        name: {
+            **fields,
+            "entries": (
+                len(variables[name]) if isinstance(variables.get(name), list) else None
+            ),
+            **(
+                {
+                    "constraints": {
+                        field: terse_constraint(rules[field])
+                        for field in entry_rules[name]
+                    }
+                }
+                if name in entry_rules
+                else {}
+            ),
+        }
+        for name, fields in list_fields(definition).items()
+    }
+    return {
+        "shape": shape,
+        "traits": traits,
+        "summary": summary,
+        "summary_truncated": truncated,
+        "lists": lists,
+        "declared": declared,
+    }
+
+
+COMPACT_FIELDS = (
+    "summary",
+    "shape",
+    "traits",
+    "cost",
+    "kinds",
+    "variable_names",
+    "lists",
+    "constraints",
+    "configures",
+)
+
+# Carried in the compact view only when set: a template has no
+# `configures`, most workflows have no list-driven step, and most declare
+# no bound on a variable
+_COMPACT_WHEN_SET = frozenset({"configures", "lists", "constraints"})
+
+
+def terse_constraint(rule):
+    """One variable's rule as a phrase, for the compact listing.
+
+    The block itself carries a `reason` in the author's words, which is what
+    a consumer reading one workflow wants and what the whole catalog cannot
+    afford - the compact listing has a token budget (#101), and the numbers
+    are the part that stops the next consumer picking 61 (#96).
+    """
+    if not isinstance(rule, dict):
+        return rule
+    parts = []
+    if rule.get("modulus"):
+        parts.append(f"{rule['modulus']}*n+{rule.get('remainder', 0)}")
+    low, high = rule.get("min_frames"), rule.get("max_frames")
+    if low is not None and high is not None:
+        parts.append(f"{low}-{high}")
+    elif low is not None:
+        parts.append(f"{low}+")
+    elif high is not None:
+        parts.append(f"up to {high}")
+    if rule.get("snap") == "up":
+        parts.append("rounds up")
+    return ", ".join(parts)
+
+
+def project_listing(
+    details,
+    *,
+    shape=None,
+    traits=None,
+    configures=None,
+    include_models=False,
+    view=None,
+):
+    """The listing an agent asked for: filtered by shape and traits, and in
+    the compact view stripped to what choosing a template needs.
+
+    Compact is templates-only unless `include_models` or `configures` says
+    otherwise - nine checkpoint variants of text-to-image are the noise the
+    two-tree split removed. A workflow with no `configures` is a template
+    for this purpose, whichever directory it sits in: a user wrote it to be
+    found. The full view never drops entries or fields.
+    """
+    if shape is not None and shape not in SHAPES:
+        raise ValueError(
+            f"Unknown shape {shape!r}. The shapes are: {', '.join(SHAPES)}."
+        )
+    traits = list(traits or [])
+    unknown = [t for t in traits if t not in TRAITS]
+    if unknown:
+        raise ValueError(
+            f"Unknown trait(s) {', '.join(unknown)}. The traits are: {', '.join(TRAITS)}."
+        )
+    if view not in (None, "compact"):
+        raise ValueError("view must be 'compact' or omitted")
+
+    compact = view == "compact"
+    keep_models = include_models or configures is not None or not compact
+    projected = {}
+    for name, detail in details.items():
+        is_model = bool(detail.get("configures") or detail.get("configures_missing"))
+        if shape is not None and detail.get("shape") != shape:
+            continue
+        if traits and not set(traits) <= set(detail.get("traits", [])):
+            continue
+        if configures is not None and detail.get("configures") != configures:
+            continue
+        if is_model and not keep_models:
+            continue
+        if compact:
+            slim = {
+                key: detail.get(key)
+                for key in COMPACT_FIELDS
+                if key not in _COMPACT_WHEN_SET or detail.get(key)
+            }
+            if slim.get("constraints"):
+                # A rule that reaches only a list-entry field is reported
+                # beside that field under `lists`, so repeating it here
+                # would state it twice in one answer - and next to
+                # `variable_names`, which does not carry the name (#145).
+                # The full listing and `get_workflow` keep the block as the
+                # author wrote it
+                in_lists = {
+                    field
+                    for entry in (slim.get("lists") or {}).values()
+                    for field in (entry.get("constraints") or {})
+                }
+                declared = set(detail.get("variable_names") or ())
+                slim["constraints"] = {
+                    variable: terse_constraint(rule)
+                    for variable, rule in slim["constraints"].items()
+                    if variable in declared or variable not in in_lists
+                }
+                if not slim["constraints"]:
+                    del slim["constraints"]
+            # This box's own history, at its two-key budget (#93/#101): the
+            # cold median, which is the one comparable to a curated `cost`,
+            # and how many runs stand behind it. The block with the
+            # cold/warm split, the drivers and `since` is in the full
+            # listing and in `get_workflow`
+            observed = detail.get("observed") or {}
+            if observed.get("cold_minutes") is not None:
+                slim["observed_minutes"] = observed["cold_minutes"]
+                slim["observed_runs"] = observed["cold_runs"]
+            if detail.get("configures_missing"):
+                slim["configures_missing"] = detail["configures_missing"]
+            projected[name] = slim
+        else:
+            projected[name] = detail
+    return projected