diffusers-workflow 0.4.0__py3-none-any.whl

dw/scalar_result_validation.py

@@ -0,0 +1,97 @@
+"""A `result` block on a step whose command returns a scalar, not an
+artifact (#212).
+
+`judge` returns a bare `float` - a natural fit for `result` to look like it
+applies (the guide's Result Configuration section describes saving text,
+and a score reads like one), but there is no file to write. `validate_workflow`
+answered `valid: true` and the run generated its full fan-out before dying
+deep inside `save_artifact` on `write() argument must be str, not float`,
+naming neither the step nor the command. Checked here against the command's
+own declared `returns` kind (`dw/tasks/task.py`'s `register_command`) rather
+than a name match on `judge`, so a future scalar-returning task is covered
+by declaring itself rather than by a second special case here.
+
+A "json" command (the assessment probes, #387) answers a dict of
+measurements, which `Result.save` writes whole only under
+`application/json`; any other content type explodes the dict key by key
+into files, or dies on a number. So a `result` on one is allowed and must
+say `application/json`.
+"""
+
+from .for_each import MEMBER_SEPARATOR, render_path
+from .tasks.task import task_command_info
+
+RESULT_KEY = "result"
+JSON_CONTENT_TYPE = "application/json"
+
+
+def scalar_result_errors(workflow_definition, source_indices=None):
+    """Every `result` block a command's `returns` kind cannot save, as
+    [{path, message}].
+
+    The definition handed here has already been substituted and expanded,
+    so a step's `task.command` is literal. `source_indices`, when given, is
+    the source step index of each step - a `for_each` group turns one
+    written step into several, and the path an error carries has to be one
+    the author can find in the file they wrote; the member is named in the
+    message.
+    """
+    steps = workflow_definition.get("steps")
+    if not isinstance(steps, list):
+        return []
+
+    errors = []
+    for index, step in enumerate(steps):
+        if not isinstance(step, dict):
+            continue
+        result = step.get(RESULT_KEY)
+        if not isinstance(result, dict):
+            continue
+        task = step.get("task")
+        if not isinstance(task, dict):
+            continue
+        command = task.get("command")
+        if not isinstance(command, str):
+            continue
+        try:
+            info = task_command_info(command)
+        except ValueError:
+            continue
+        returns = info.get("returns")
+        if returns == "json":
+            if result.get("content_type") == JSON_CONTENT_TYPE:
+                continue
+            message = (
+                f"{command} answers a JSON document - 'result' must set "
+                f"content_type '{JSON_CONTENT_TYPE}', not "
+                f"{result.get('content_type')!r}"
+            )
+        elif returns == "scalar":
+            message = (
+                f"{command} returns a number, not an artifact - "
+                f"'result' cannot be saved"
+            )
+        else:
+            continue
+
+        source = (
+            source_indices[index]
+            if source_indices is not None and index < len(source_indices)
+            else index
+        )
+        name = step.get("name")
+        where = (
+            f" in member '{name}'"
+            if isinstance(name, str) and MEMBER_SEPARATOR in name
+            else ""
+        )
+        errors.append(
+            {
+                "path": render_path(("steps", source, RESULT_KEY)),
+                "message": f"{message}{where}",
+            }
+        )
+    return errors
+
+
+__all__ = ["scalar_result_errors"]

dw/schema.py

@@ -0,0 +1,283 @@
+import json
+import os
+from jsonschema import validate, ValidationError
+from jsonschema.exceptions import best_match
+from jsonschema.validators import validator_for
+
+
+def validate_data(data, schema):
+    try:
+        validate(instance=data, schema=schema)
+        return True, "Validation successful"
+
+    except ValidationError as ve:
+        path = json_path(ve.absolute_path)
+        location = f" at {path}" if path else ""
+        return False, f"Validation error{location}: {error_message(ve)}"
+    except json.JSONDecodeError as je:
+        return False, f"JSON parsing error: {str(je)}"
+    except Exception as e:
+        return False, f"Unexpected error: {str(e)}"
+
+
+# anyOf branches produce dozens of near-identical entries; this is more than
+# an agent fixes in one pass, and enough that nothing real is hidden
+MAX_VALIDATION_ERRORS = 25
+
+
+def validate_data_all(data, schema):
+    """Every schema violation in `data`, as [{path, message}].
+
+    Sorted by path, deduplicated on (path, message), capped at
+    MAX_VALIDATION_ERRORS. Each top-level error is reduced with best_match
+    - the same descent into anyOf branches jsonschema.validate performs to
+    pick the one exception it raises - so a definition with a single
+    violation is reported exactly as validate_data reports it.
+    """
+    validator = validator_for(schema)(schema)
+    seen = {}
+    for error in validator.iter_errors(data):
+        chosen = best_match([error])
+        key = (json_path(chosen.absolute_path), error_message(chosen))
+        seen.setdefault(key, None)
+    ordered = sorted(
+        _only_unknown_property(seen), key=lambda key: (key[0] or "", key[1])
+    )
+    return [
+        {"path": path, "message": message}
+        for path, message in ordered[:MAX_VALIDATION_ERRORS]
+    ]
+
+
+def _only_unknown_property(keys):
+    """Drop the shape complaints about a key that is simply unknown.
+
+    A key refused by a typed `additionalProperties` fails that shape's own
+    'type' and 'required' checks too, so one mistyped 'trasformer' arrives
+    as three errors, two of which describe the component definition the
+    author never meant to write. Where a path has the unknown-property
+    message, it is the whole story (#123).
+    """
+    unknown = {
+        path for path, message in keys if message.startswith('unknown property "')
+    }
+    return [
+        (path, message)
+        for path, message in keys
+        if path not in unknown or message.startswith('unknown property "')
+    ]
+
+
+def error_message(error):
+    """The message an agent can act on for one schema violation.
+
+    Only `additionalProperties` is rewritten. jsonschema says "Additional
+    properties are not allowed ('when' was unexpected)", which does not say
+    what the object *does* take - and the whole point of closing these
+    objects is that an invented key ('when', 'retry') or a mistyped real one
+    ('relase_pipeline') is a silent no-op the engine never reads (#118). So
+    the message names the object, the key, and the full set that is read.
+    """
+    if error.validator != "additionalProperties":
+        # An object that is closed except for one shape of key - the pipeline,
+        # where any other key may name a component - refuses an unknown key
+        # through that shape's own 'type'/'required' rather than through
+        # additionalProperties, and jsonschema then reports the shape rather
+        # than the key. The schema carries the sentence to say instead (#123)
+        explanation = (error.schema or {}).get("unknownPropertyMessage")
+        if explanation and error.absolute_path:
+            return f'unknown property "{error.absolute_path[-1]}" - {explanation}'
+        return error.message
+
+    properties = (error.schema or {}).get("properties") or {}
+    # An engine-injected key is legal but is not something an author writes,
+    # so listing it as one of the properties on offer only invites its use
+    allowed = sorted(
+        name
+        for name, subschema in properties.items()
+        if not str((subschema or {}).get("description", "")).startswith(
+            "Engine-injected"
+        )
+    )
+    instance = error.instance if isinstance(error.instance, dict) else {}
+    unknown = sorted(key for key in instance if key not in allowed)
+    name = instance.get("name")
+    named = f'"{name}": ' if isinstance(name, str) and name else ""
+    keys = ", ".join(f'"{key}"' for key in unknown) or "a property"
+    plural = "properties" if len(unknown) > 1 else "property"
+    return (
+        f"{named}unknown {plural} {keys} - the engine reads only "
+        f"{', '.join(allowed)}, so anything else would be silently ignored"
+    )
+
+
+def format_validation_errors(errors):
+    """The message a raised validation failure carries.
+
+    One error keeps the line every caller already shows -
+    'Validation error at <path>: <message>'. Several are listed one per
+    line under a single heading, so the text 'Validation error' still
+    appears once per failure (the CLI and the REPL count on that).
+    """
+    if len(errors) == 1:
+        path, message = errors[0]["path"], errors[0]["message"]
+        location = f" at {path}" if path else ""
+        return f"Validation error{location}: {message}"
+    count = (
+        f"first {MAX_VALIDATION_ERRORS}"
+        if len(errors) >= MAX_VALIDATION_ERRORS
+        else str(len(errors))
+    )
+    lines = [f"Validation errors ({count}):"]
+    for error in errors:
+        lines.append(f"  at {error['path'] or 'root'}: {error['message']}")
+    return "\n".join(lines)
+
+
+def json_path(absolute_path):
+    """Render a jsonschema ValidationError's absolute_path (a deque of dict
+    keys and list indices) as a dotted/bracket path, e.g.
+    steps[0].pipeline.arguments.prompt. None if the error is at the root."""
+    if not absolute_path:
+        return None
+    parts = []
+    for element in absolute_path:
+        if isinstance(element, int):
+            parts.append(f"[{element}]")
+        elif not parts:
+            parts.append(str(element))
+        else:
+            parts.append(f".{element}")
+    return "".join(parts)
+
+
+def load_schema(schema_name):
+    file_spec = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), f"{schema_name}_schema.json"
+    )
+    with open(file_spec, "r") as file:
+        return json.load(file)
+
+
+# The parts of the workflow schema that can be asked for on their own, and
+# what each holds. Whole, the schema is ~36 KB - ~8.6k tokens in one call,
+# spent by an agent that needed the shape of a `result` block (#101). A
+# section answers that question at a tenth the size; the no-argument call
+# is unchanged, so nothing that reads the whole schema is affected.
+SCHEMA_SECTIONS = {
+    "variables": {
+        "properties": [
+            "id",
+            "description",
+            "summary",
+            "shape",
+            "traits",
+            "configures",
+            "cost",
+            "cost_drivers",
+            "variables",
+            "variable_constraints",
+            "seed",
+        ],
+        "defs": ["variable_constraint"],
+    },
+    "steps": {"properties": ["steps"], "defs": ["step", "workflow_reference"]},
+    "pipelines": {
+        "properties": [],
+        "defs": ["pipeline", "pipeline_reference", "chain", "arguments", "image"],
+    },
+    "tasks": {"properties": [], "defs": ["task"]},
+    "result": {"properties": [], "defs": ["result"]},
+    "configuration": {
+        "properties": [],
+        "defs": [
+            "pipeline_configuration",
+            "pipeline_component",
+            "shared_components",
+            "reused_components",
+            "quantization_config",
+            "scheduler",
+            "lora",
+            "controlnet",
+            "ip_adapter",
+            "from_pretrained_arguments",
+            "group_offload",
+            "compile_config",
+            "enable_layerwise_casting",
+        ],
+    },
+}
+
+DEFS_KEY = "$defs"
+DEFS_REF_PREFIX = f"#/{DEFS_KEY}/"
+
+
+class SchemaSectionError(LookupError):
+    """A section name the schema has no part for. The message names the
+    ones it does, so a route can hand it back as a 404 detail."""
+
+
+def _referenced_defs(node, found):
+    """Every '#/$defs/x' name reachable from a subtree, into `found`."""
+    if isinstance(node, dict):
+        ref = node.get("$ref")
+        if isinstance(ref, str) and ref.startswith(DEFS_REF_PREFIX):
+            found.add(ref[len(DEFS_REF_PREFIX) :])
+        for value in node.values():
+            _referenced_defs(value, found)
+    elif isinstance(node, list):
+        for value in node:
+            _referenced_defs(value, found)
+    return found
+
+
+def schema_section(schema, section):
+    """One part of the workflow schema, plus where the rest of it is.
+
+    The fragment carries only the definitions this section owns; a `$ref`
+    to a definition another section owns is left standing and named in
+    `elsewhere` ({definition: section}), because expanding it transitively
+    would pull most of the schema back in and there would be no section
+    left to speak of. `sections` lists every section name.
+
+    Raises:
+        SchemaSectionError: If `section` is not one of SCHEMA_SECTIONS.
+    """
+    if section not in SCHEMA_SECTIONS:
+        raise SchemaSectionError(
+            f"No schema section '{section}'. The sections are: "
+            f"{', '.join(sorted(SCHEMA_SECTIONS))}."
+        )
+    wanted = SCHEMA_SECTIONS[section]
+    defs = schema.get(DEFS_KEY, {})
+    fragment = {
+        key: schema[key]
+        for key in ("$schema", "$id", "type", "description")
+        if key in schema
+    }
+    properties = {
+        name: schema["properties"][name]
+        for name in wanted["properties"]
+        if name in schema.get("properties", {})
+    }
+    if properties:
+        fragment["properties"] = properties
+        required = [name for name in schema.get("required", []) if name in properties]
+        if required:
+            fragment["required"] = required
+    held = {name: defs[name] for name in wanted["defs"] if name in defs}
+    if held:
+        fragment[DEFS_KEY] = held
+    owner = {
+        name: other for other, part in SCHEMA_SECTIONS.items() for name in part["defs"]
+    }
+    referenced = _referenced_defs(fragment, set())
+    elsewhere = {
+        name: owner[name] for name in sorted(referenced - set(held)) if name in owner
+    }
+    return {
+        "section": section,
+        "sections": sorted(SCHEMA_SECTIONS),
+        "elsewhere": elsewhere,
+        "schema": fragment,
+    }