gen3-dataops-toolkit 4.1.0__tar.gz → 4.3.0__tar.gz

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gen3-dataops-toolkit
-Version: 4.1.0
+Version: 4.3.0
 Summary: Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments
 License: Apache-2.0
 Author: JoshuaHarris391
@@ -194,6 +194,20 @@ version until the CDK config catches up —
 `g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
 reports exactly that gap and exits 1, so it can gate CI.
 
+### Deleting metadata: where the node order comes from
+
+`delete metadata` walks nodes children-before-parents. The order is resolved
+from the first available of: an explicit `--import-order <path|s3://…>`
+(failures are fatal — an explicit source is never silently skipped), the
+registered study's release bucket (`DataImportOrder.txt` next to the release's
+node JSONs), a `DataImportOrder.txt` in the current directory, or a
+topological sort derived from the dictionary itself — the `--dict-version`
+tag's bundle when given (downloaded to `~/.g3dt/schemas` if needed), else the
+env's deployed dictionary. Deriving matters when deleting data submitted
+under an older dictionary whose node layout differs from today's: pass
+`--dict-version <old-tag>`. With `--on ec2`, `--import-order` must be an
+`s3://` URI (a laptop path does not exist on the box).
+
 Synthetic data is only schema-valid against the dictionary that generated it, so
 `synth generate` records the dictionary version in each batch and `synth upload`
 refuses a batch that doesn't match the version being uploaded (override with

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.3.0}/README.md

@@ -160,6 +160,20 @@ version until the CDK config catches up —
 `g3dt config diff --env <env> --file <wrapper>/config/<project>.<env>.json`
 reports exactly that gap and exits 1, so it can gate CI.
 
+### Deleting metadata: where the node order comes from
+
+`delete metadata` walks nodes children-before-parents. The order is resolved
+from the first available of: an explicit `--import-order <path|s3://…>`
+(failures are fatal — an explicit source is never silently skipped), the
+registered study's release bucket (`DataImportOrder.txt` next to the release's
+node JSONs), a `DataImportOrder.txt` in the current directory, or a
+topological sort derived from the dictionary itself — the `--dict-version`
+tag's bundle when given (downloaded to `~/.g3dt/schemas` if needed), else the
+env's deployed dictionary. Deriving matters when deleting data submitted
+under an older dictionary whose node layout differs from today's: pass
+`--dict-version <old-tag>`. With `--on ec2`, `--import-order` must be an
+`s3://` URI (a laptop path does not exist on the box).
+
 Synthetic data is only schema-valid against the dictionary that generated it, so
 `synth generate` records the dictionary version in each batch and `synth upload`
 refuses a batch that doesn't match the version being uploaded (override with

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gen3-dataops-toolkit"
-version = "4.1.0"
+version = "4.3.0"
 description = "Gen3 DataOps toolkit (g3dt): operate SSM-published Gen3 data pipeline environments"
 authors = ["JoshuaHarris391 <harjo391@gmail.com>"]
 readme = "README.md"
@@ -44,7 +44,7 @@ moto = ">=5.0"
 optional = true
 
 [tool.poetry.group.synth.dependencies]
-gen3-metadata-simulator = "^0.3.0"
+gen3-metadata-simulator = "^0.5.3"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/cli/delete_cmds.py

@@ -7,6 +7,19 @@ the bare names (a specific version like ``0.9.8``, resolved via an Athena GUID
 lookup, or ``all`` for every version). A bare study with no version anywhere
 is refused (exit 2).
 
+``--synthetic`` switches to registry-free mode for synthetic data: each
+``--studies`` name is the Gen3 project code itself (no SSM study registry —
+synthetic projects are never registered), bare names default to version
+``all``, and a specific version is matched verbatim against the records'
+``data_version`` property via GraphQL rather than Athena receipts (synthetic
+uploads write none).
+
+The node deletion order comes from, in order: an explicit ``--import-order``
+(path or s3:// URI; failures are fatal, never silently skipped), the
+registered study's release bucket, a ``DataImportOrder.txt`` in the current
+directory, or a topological sort derived from the dictionary itself — the
+``--dict-version`` bundle when given, else the env's deployed dictionary.
+
 Every command confirms before acting. Production always requires typing the
 target id, even with ``--yes``. Deleting ALL versions always prompts, even with
 ``--yes``. Confirmation happens locally before any EC2 dispatch (SSM has no
@@ -61,12 +74,30 @@ def _normalise_version(raw: str, where: str) -> str:
     return match.group(1)
 
 
-def _parse_study_specs(studies: str, fallback, env: str):
+def _synthetic_version(raw: str) -> str:
+    """Canonicalise a synthetic version token: ``all`` (any case) or verbatim.
+
+    Synthetic versions are matched exactly against the records' ``data_version``
+    property, and the natural label there is the dictionary version WITH its
+    leading ``v`` (batch dirs are ``~/.g3dt/synth_metadata/v1.3.0/...``) — so
+    unlike ``_normalise_version`` nothing is stripped. A version that matches
+    no records is reported by the worker (skip + hint), not silently absorbed.
+    """
+    token = raw.strip()
+    return "all" if token.lower() == "all" else token
+
+
+def _parse_study_specs(studies: str, fallback, env: str, synthetic: bool = False):
     """Turn ``--studies`` into ``[(resolved_study_key, version), ...]``.
 
     Each comma-separated entry is ``name`` or ``name:version``. A bare name
     takes *fallback* (the ``--version`` default); *fallback* is ``None`` when
-    ``--version`` was not given, which makes a bare name a usage error.
+    ``--version`` was not given, which makes a bare name a usage error —
+    except with *synthetic*, where a bare name defaults to ``all`` (the whole
+    point of the flag is "wipe the synthetic project").
+
+    With *synthetic* the raw name IS the Gen3 project code: no study-registry
+    lookup, and version tokens pass through :func:`_synthetic_version`.
 
     Every entry is validated before anything is dispatched, so a typo in the
     last study cannot leave the earlier ones already deleted.
@@ -102,9 +133,15 @@ def _parse_study_specs(studies: str, fallback, env: str):
             raise typer.Exit(2)
 
         if sep:
-            version = _normalise_version(raw_version, f"for study '{name}'")
+            version = (
+                _synthetic_version(raw_version)
+                if synthetic
+                else _normalise_version(raw_version, f"for study '{name}'")
+            )
         elif fallback is not None:
             version = fallback
+        elif synthetic:
+            version = "all"
         else:
             typer.secho(
                 f"No version for study '{name}': add ':<version>' to it "
@@ -115,7 +152,7 @@ def _parse_study_specs(studies: str, fallback, env: str):
             )
             raise typer.Exit(2)
 
-        specs.append((study_of(name, env).key, version))
+        specs.append((name if synthetic else study_of(name, env).key, version))
 
     if not specs:
         typer.secho("--studies is empty.", fg=typer.colors.RED, err=True)
@@ -141,6 +178,35 @@ def metadata(
              "':version', e.g. 0.9.8, or 'all' for every version.",
     ),
     node: Optional[str] = typer.Option(None, "--node", help="Delete only this node type."),
+    synthetic: bool = typer.Option(
+        False,
+        "--synthetic",
+        help="Registry-free synthetic-data mode: each --studies name is the "
+             "Gen3 project id itself (no SSM study registry). Bare names "
+             "default to version 'all'; a specific version matches records' "
+             "data_version property verbatim.",
+    ),
+    program_id: Optional[str] = typer.Option(
+        None,
+        "--program-id",
+        help="Gen3 program for --synthetic (default: program1). "
+             "Invalid without --synthetic.",
+    ),
+    import_order: Optional[str] = typer.Option(
+        None,
+        "--import-order",
+        help="Path or s3:// URI of DataImportOrder.txt. Default: auto — the "
+             "study's release bucket (registered studies), then "
+             "./DataImportOrder.txt, then derived from the dictionary. "
+             "With --on ec2 only s3:// URIs are accepted.",
+    ),
+    dict_version: Optional[str] = typer.Option(
+        None,
+        "--dict-version",
+        help="Dictionary git tag to derive the node order from (verbatim, "
+             "e.g. v1.3.0). Default: the env's deployed dictionary. Only "
+             "used when the order is derived.",
+    ),
     yes: bool = typer.Option(
         False, "--yes", "-y", help="Skip the non-prod prompt (specific-version only)."
     ),
@@ -156,12 +222,44 @@ def metadata(
 
       g3dt delete metadata --studies "ausdiab:0.7.5,cdah:0.8.1" --env staging
       g3dt delete metadata --studies "ausdiab:all,cdah" --version 0.9.8 --env staging
+      g3dt delete metadata --studies "synthetic_dataset_1,synthetic_dataset_2" --env test --synthetic
     """
     env = resolve.active_env(env)
-    fallback = (
-        _normalise_version(version, "for --version") if version is not None else None
-    )
-    specs = _parse_study_specs(studies, fallback, env)
+    if program_id is not None and not synthetic:
+        typer.secho(
+            "--program-id is only valid with --synthetic (registered studies "
+            "carry their program in the study registry).",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    if import_order and dict_version:
+        typer.secho(
+            "--import-order names the exact file; --dict-version derives one "
+            "— pass only one.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    # A laptop path forwarded to the EC2 box would resolve against the box's
+    # filesystem — at best a crash after confirmation, at worst a same-named
+    # DIFFERENT file ordering the delete. s3:// URIs (and --dict-version)
+    # resolve identically anywhere, so only those may travel.
+    if on == Target.ec2 and import_order and not import_order.startswith("s3://"):
+        typer.secho(
+            "--import-order with --on ec2 must be an s3:// URI (a local path "
+            "does not exist on the box). Upload the file, or run locally.",
+            fg=typer.colors.RED,
+            err=True,
+        )
+        raise typer.Exit(2)
+    if version is None:
+        fallback = None
+    elif synthetic:
+        fallback = _synthetic_version(version)
+    else:
+        fallback = _normalise_version(version, "for --version")
+    specs = _parse_study_specs(studies, fallback, env, synthetic=synthetic)
     versions = [v for _, v in specs]
 
     # The typed production confirmation stays the study keys alone: short
@@ -171,13 +269,17 @@ def metadata(
     uniform = len(set(versions)) == 1
     any_all = "all" in versions
 
+    prefix = "synthetic " if synthetic else ""
     if uniform and versions[0] == "all":
-        action = "deletion of ALL VERSIONS"
+        action = f"{prefix}deletion of ALL VERSIONS"
     elif uniform:
-        action = f"deletion of v{versions[0]}"
+        # Synthetic versions are verbatim data_version values (often already
+        # v-prefixed); Athena versions are canonical x.y.z, displayed with v.
+        shown = versions[0] if synthetic else f"v{versions[0]}"
+        action = f"{prefix}deletion of {shown}"
     else:
         plan = ", ".join(f"{key}:{v}" for key, v in specs)
-        action = f"deletion of per-study versions [{plan}]"
+        action = f"{prefix}deletion of per-study versions [{plan}]"
 
     # Deleting every version is the most destructive path: always prompt (pass
     # assume_yes=False so --yes can't bypass it; prod still types the target).
@@ -200,6 +302,15 @@ def metadata(
             ]
         if node:
             a += ["--node", node]
+        if synthetic:
+            # Program is always passed explicitly: the shell default makes it
+            # optional on the wire, but an explicit value keeps the contract
+            # visible in logs and SSM command history.
+            a += ["--synthetic", "--program-id", program_id or "program1"]
+        if import_order:
+            a += ["--import-order", import_order]
+        if dict_version:
+            a += ["--dict-version", dict_version]
         return a
 
     def remote_cli(env_name):
@@ -212,6 +323,17 @@ def metadata(
         a.append("--yes")
         if node:
             a += ["--node", node]
+        if synthetic:
+            # Without this the remote re-entry would re-parse --studies
+            # against the study registry on the box and exit 2.
+            a.append("--synthetic")
+            if program_id is not None:
+                a += ["--program-id", program_id]
+        if import_order:
+            # Guaranteed s3:// by the pre-dispatch gate above.
+            a += ["--import-order", import_order]
+        if dict_version:
+            a += ["--dict-version", dict_version]
         return a
 
     dispatch.run_or_dispatch(

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/cli/synth.py

@@ -370,6 +370,13 @@ def generate(
         None, "--version", "-v",
         help="Version label for output dir (default: env dictionary_version).",
     ),
+    data_version: Optional[str] = typer.Option(
+        None, "--data-version",
+        help="Stamp every generated record's data_version property with this "
+             "value (requires the dictionary to declare data_version). Makes "
+             "the batch deletable later with "
+             "'delete metadata --synthetic --version <value>'.",
+    ),
 ) -> None:
     """Generate synthetic metadata locally with gen3-metadata-simulator.
 
@@ -431,6 +438,8 @@ def generate(
         args += ["--num-records", num_records]
     if seed is not None:
         args += ["--seed", str(seed)]
+    if data_version:
+        args += ["--data-version", data_version]
     env_vars = script_env(e, ver)
     if effective_provider is Provider.llm:
         env_vars.update(

{gen3_dataops_toolkit-4.1.0 → gen3_dataops_toolkit-4.3.0}/src/g3dt/config.py

@@ -40,6 +40,7 @@ import re
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Dict, List, Optional, Tuple
+from urllib.parse import unquote, urlsplit
 
 import yaml
 
@@ -393,6 +394,8 @@ class EnvConfig:
     dictionary_version: str
     aws_profile: Optional[str]
     aws_secret_name: str
+    # Canonical scheme-less "bucket/key" (normalize_s3_location at resolve time;
+    # callers prepend s3:// themselves).
     schema_s3_uri: str
     domain: str
     app_name: str
@@ -421,6 +424,74 @@ def _app_or_default(rc, leaf: str, default: str) -> str:
     return (rc.get(f"app/{leaf}") or "").strip() or default
 
 
+#: The two S3 endpoint URL host styles. Path-style must be tried first: a bare
+#: ``s3.<region>.amazonaws.com`` host would otherwise match the virtual-hosted
+#: pattern with bucket "s3". The virtual-hosted bucket group is greedy so
+#: dotted bucket names keep their dots.
+_S3_PATH_STYLE_HOST = re.compile(r"^s3([.-][a-z0-9-]+)*\.amazonaws\.com$")
+_S3_VIRTUAL_HOST = re.compile(r"^(?P<bucket>.+)\.s3([.-][a-z0-9-]+)*\.amazonaws\.com$")
+
+#: Remediation appended to normalize_s3_location errors for the SSM app fact.
+_SCHEMA_S3_URI_HINT = (
+    "Fix gen3.schemaS3Uri in the CDK config (gen3-aws-data-pipeline) and "
+    "re-run `cdk deploy`."
+)
+
+
+def normalize_s3_location(
+    value: str, *, param: str = "app/schema_s3_uri", hint: Optional[str] = None
+) -> str:
+    """Canonicalize an operator-supplied S3 location to scheme-less ``bucket[/key]``.
+
+    Accepted forms: ``bucket/key`` (canonical), ``s3://bucket/key`` — a repeated
+    scheme (``s3://s3://...``) is tolerated, since that is exactly what a
+    scheme-carrying SSM value produces once callers prepend ``s3://`` — and the
+    two S3 endpoint URL styles (``https://<bucket>.s3.<region>.amazonaws.com/<key>``,
+    ``https://s3.<region>.amazonaws.com/<bucket>/<key>``); query strings on URL
+    forms (presigned links, ``?versionId=``) are dropped.
+
+    An object key is NOT required — ``resolve_env`` gates every command, so
+    shape is enforced at point of use (upload). The key is preserved verbatim,
+    trailing slash included, except for percent-decoding of URL forms.
+
+    :raises ConfigError: empty value, unrecognizable http(s) host (e.g. an AWS
+        console page URL), or a bucket segment containing ``:`` (the
+        ``s3:/bucket`` one-slash typo).
+    """
+    original = value.strip()
+
+    def _bad() -> ConfigError:
+        msg = (
+            f"Cannot interpret {param} value '{original}' as an S3 location. "
+            "Accepted forms: bucket/key, s3://bucket/key, or an S3 endpoint "
+            "URL (https://<bucket>.s3.<region>.amazonaws.com/<key> or "
+            "https://s3.<region>.amazonaws.com/<bucket>/<key>)."
+        )
+        return ConfigError(msg + (f" {hint}" if hint else ""))
+
+    v = original
+    if not v:
+        raise _bad()
+    while v.lower().startswith("s3://"):
+        v = v[len("s3://") :]
+    if v.lower().startswith(("https://", "http://")):
+        parts = urlsplit(v)
+        host = parts.hostname or ""
+        path = unquote(parts.path)
+        if _S3_PATH_STYLE_HOST.match(host):
+            v = path.lstrip("/")
+        else:
+            m = _S3_VIRTUAL_HOST.match(host)
+            if not m:
+                raise _bad()
+            key = path.lstrip("/")
+            v = f"{m.group('bucket')}/{key}" if key else m.group("bucket")
+    bucket = v.split("/", 1)[0]
+    if not bucket or ":" in bucket:
+        raise _bad()
+    return v
+
+
 def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
     """Resolve one environment: app INPUT facts + CDK OUTPUT names, from SSM.
 
@@ -454,7 +525,9 @@ def resolve_env(env: str, project: Optional[str] = None) -> EnvConfig:
         dictionary_version=rc.app("dictionary_version"),
         aws_profile=profile,
         aws_secret_name=rc.app("aws_secret_name"),
-        schema_s3_uri=rc.app("schema_s3_uri"),
+        schema_s3_uri=normalize_s3_location(
+            rc.app("schema_s3_uri"), hint=_SCHEMA_S3_URI_HINT
+        ),
         domain=rc.app("domain"),
         app_name=rc.app("app_name"),
         namespace=rc.app("namespace"),

gen3_dataops_toolkit-4.3.0/src/g3dt/import_order.py

@@ -0,0 +1,265 @@
+"""Node import-order resolution for metadata deletion.
+
+Deleting Gen3 metadata must walk nodes children-before-parents, which the
+toolkit historically took from a ``DataImportOrder.txt`` in the caller's
+working directory — a silent dependency that crashed runs started anywhere
+else. This module resolves the order from an explicit chain of sources:
+
+1. an explicit ``--import-order`` (local path or ``s3://`` URI) — fatal if
+   unreadable, never silently skipped;
+2. the study's release bucket (registered studies only — releases ship a
+   ``DataImportOrder.txt`` next to the node JSONs);
+3. a ``DataImportOrder.txt`` in the current directory (legacy behavior);
+4. derivation from the dictionary itself: a topological sort over the raw
+   bundle's ``links`` (proven byte-identical to simulator-written order
+   files), reading either the deployed dictionary at ``schema_s3_uri`` or a
+   ``--dict-version`` bundle cached under ``~/.g3dt/schemas``.
+
+Deliberately NOT used: ``gen3_validator``'s ``DataDictionary.get_node_order``
+(it force-moves core_metadata_collection last, which a deletion reversal would
+delete FIRST — before the file nodes that link to it) and its
+``get_node_link`` (inspects only ``links[0]`` for subgroups).
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.error
+import urllib.request
+from collections import deque
+from pathlib import Path
+from typing import List, Optional, Sequence, Tuple
+
+logger = logging.getLogger(__name__)
+
+#: Bundle keys that are shared definitions, not nodes.
+_META_KEYS = frozenset(
+    {"_definitions.yaml", "_terms.yaml", "_settings.yaml", "root.yaml", "metaschema.yaml"}
+)
+
+_ORDER_FILENAME = "DataImportOrder.txt"
+
+
+class ImportOrderError(Exception):
+    """A node order could not be resolved; the message names the fix."""
+
+
+def derive_import_order(schema: dict) -> List[str]:
+    """Topologically sort a raw Gen3 dictionary bundle into submission order.
+
+    Works on the UNRESOLVED bundle ({"<node>.yaml": {...}, ...}): ``links``
+    blocks are self-contained (no $refs), so no schema resolution is needed.
+    Skips the shared ``_*.yaml`` definitions, entries without ``properties``,
+    and ``submittable: false`` nodes (which removes ``program``). Every
+    ``links`` entry is flattened — subgroup members and plain links alike —
+    into edges ``parent -> child``; edges to nodes outside the set are
+    dropped (this handles ``project -> program``).
+
+    Kahn's algorithm with sorted tie-breaking keeps the output deterministic
+    and byte-compatible with simulator-written DataImportOrder.txt files.
+    Members of a dependency cycle (impossible in a valid Gen3 dictionary) are
+    appended at the end with a warning rather than dropped.
+
+    Returns SUBMISSION order (parents first); callers reverse for deletion.
+    """
+    nodes = {}
+    for key, value in schema.items():
+        if key in _META_KEYS or not isinstance(value, dict):
+            continue
+        if "properties" not in value or value.get("submittable") is False:
+            continue
+        name = key[: -len(".yaml")] if key.endswith(".yaml") else key
+        nodes[name] = value
+
+    graph = {n: [] for n in nodes}
+    in_degree = {n: 0 for n in nodes}
+    for name, node in nodes.items():
+        for entry in node.get("links", []) or []:
+            members = entry.get("subgroup", [entry]) if isinstance(entry, dict) else []
+            for member in members:
+                parent = member.get("target_type")
+                if parent in nodes:
+                    graph[parent].append(name)
+                    in_degree[name] += 1
+
+    queue = deque(sorted(n for n, d in in_degree.items() if d == 0))
+    ordered: List[str] = []
+    while queue:
+        node = queue.popleft()
+        ordered.append(node)
+        for child in sorted(graph[node]):
+            in_degree[child] -= 1
+            if in_degree[child] == 0:
+                queue.append(child)
+        queue = deque(sorted(queue))
+
+    leftovers = sorted(n for n in nodes if n not in ordered)
+    if leftovers:
+        logger.warning(
+            "Dictionary link cycle detected; appending unordered node(s): %s "
+            "— deleting them may require a re-run.",
+            ", ".join(leftovers),
+        )
+        ordered.extend(leftovers)
+    return ordered
+
+
+def ensure_local_dictionary(env_cfg, version: str, schema_dir: Optional[Path] = None) -> Path:
+    """Return the local bundle for ``version``, downloading it if needed.
+
+    The cache is ``$G3DT_SCHEMA_DIR`` (else ``~/.g3dt/schemas``), shared with
+    pull_dict.sh. A cache hit requires a non-empty file that parses as JSON —
+    ``wget -O`` leaves a zero-byte file behind on a failed pull, so mere
+    existence is not trusted. Downloads are validated as JSON BEFORE being
+    written, to a temp name replaced atomically, so a failed or partial
+    download can never poison the cache.
+    """
+    from g3dt import config
+
+    if schema_dir is None:
+        schema_dir = Path(
+            os.environ.get("G3DT_SCHEMA_DIR", "~/.g3dt/schemas")
+        ).expanduser()
+    target = schema_dir / config.dictionary_filename(env_cfg, version)
+
+    if target.exists() and target.stat().st_size > 0:
+        try:
+            json.loads(target.read_text(encoding="utf-8"))
+            return target
+        except ValueError:
+            logger.warning("Cached dictionary %s is corrupt; re-downloading.", target)
+
+    url = config.dictionary_url(env_cfg, version)
+    logger.info("Downloading dictionary %s from %s", version, url)
+    try:
+        with urllib.request.urlopen(url) as resp:
+            body = resp.read()
+    except (urllib.error.HTTPError, urllib.error.URLError) as exc:
+        raise ImportOrderError(
+            f"Could not download dictionary tag '{version}' from {url}: {exc}. "
+            f"Check the tag exists in the schema repo, or pass --import-order."
+        )
+    try:
+        json.loads(body.decode("utf-8"))
+    except ValueError:
+        raise ImportOrderError(
+            f"Downloaded dictionary tag '{version}' from {url} is not valid "
+            f"JSON — is the tag/path right?"
+        )
+    schema_dir.mkdir(parents=True, exist_ok=True)
+    tmp = target.with_name(target.name + ".part")
+    tmp.write_bytes(body)
+    os.replace(tmp, target)
+    return target
+
+
+def to_deletion_order(nodes: Sequence[str], exclude_nodes: Sequence[str]) -> List[str]:
+    """Filter excluded nodes out of a submission order and reverse it."""
+    kept = [n for n in nodes if n not in exclude_nodes]
+    kept.reverse()
+    return kept
+
+
+def resolve_import_order(
+    *,
+    env_cfg,
+    session,
+    import_order: Optional[str] = None,
+    dict_version: Optional[str] = None,
+    study_cfg=None,
+    cwd: Optional[Path] = None,
+) -> Tuple[List[str], str]:
+    """Resolve the node SUBMISSION order for a delete, returning (nodes, source).
+
+    The chain (first hit wins): explicit ``import_order`` (fatal on failure —
+    an explicit source is never silently skipped), the registered study's
+    release bucket (``study_cfg.s3_metadata_path``; a listing error warns and
+    falls through), a ``DataImportOrder.txt`` in the current directory
+    (legacy behavior), then derivation from the dictionary — the
+    ``dict_version`` bundle when given, else the deployed dictionary at
+    ``s3://{env_cfg.schema_s3_uri}``.
+
+    ``source`` is a human-readable description of the winning step; workers
+    log it so operators can always see what ordered a destructive run.
+    """
+    from g3dt.upload.metadata_submitter import (
+        find_data_import_order_file_s3,
+        read_data_import_order_txt_s3,
+        read_metadata_json_s3,
+    )
+
+    # 1. Explicit flag: local path or s3:// URI. Failures are fatal.
+    if import_order:
+        if import_order.startswith("s3://"):
+            try:
+                nodes = read_data_import_order_txt_s3(import_order, session)
+            except Exception as exc:
+                raise ImportOrderError(
+                    f"Could not read --import-order {import_order}: {exc}"
+                )
+            return nodes, f"explicit --import-order {import_order}"
+        path = Path(import_order)
+        try:
+            lines = path.read_text(encoding="utf-8").splitlines()
+        except OSError as exc:
+            raise ImportOrderError(
+                f"Could not read --import-order {import_order}: {exc}"
+            )
+        return (
+            [line.strip() for line in lines if line.strip()],
+            f"explicit --import-order {path}",
+        )
+
+    # 2. Registered study: the release ships its own DataImportOrder.txt.
+    if study_cfg is not None and str(
+        getattr(study_cfg, "s3_metadata_path", "")
+    ).startswith("s3://"):
+        try:
+            uri = find_data_import_order_file_s3(
+                s3_uri=study_cfg.s3_metadata_path, session=session
+            )
+            return (
+                read_data_import_order_txt_s3(uri, session),
+                f"release bucket {uri}",
+            )
+        except Exception as exc:
+            logger.warning(
+                "No usable DataImportOrder.txt under %s (%s) — falling back.",
+                study_cfg.s3_metadata_path,
+                exc,
+            )
+
+    # 3. Legacy: a DataImportOrder.txt in the working directory.
+    local = (cwd or Path.cwd()) / _ORDER_FILENAME
+    if local.exists():
+        lines = local.read_text(encoding="utf-8").splitlines()
+        return (
+            [line.strip() for line in lines if line.strip()],
+            f"{_ORDER_FILENAME} in current directory ({local.resolve()})",
+        )
+
+    # 4. Derive from the dictionary.
+    if dict_version:
+        bundle_path = ensure_local_dictionary(env_cfg, dict_version)
+        schema = json.loads(bundle_path.read_text(encoding="utf-8"))
+        source = f"derived from dictionary {dict_version} ({bundle_path})"
+    else:
+        uri = f"s3://{env_cfg.schema_s3_uri}"
+        try:
+            schema = read_metadata_json_s3(uri, session)
+        except Exception as exc:
+            raise ImportOrderError(
+                f"No DataImportOrder.txt found and the deployed dictionary at "
+                f"{uri} (SSM app/schema_s3_uri) could not be read: {exc}. "
+                f"Pass --import-order <path|s3://...> or --dict-version <tag>."
+            )
+        source = f"derived from deployed dictionary {uri}"
+
+    nodes = derive_import_order(schema)
+    if not nodes:
+        raise ImportOrderError(
+            f"Deriving the node order produced no submittable nodes — is the "
+            f"object read for '{source}' a Gen3 dictionary bundle?"
+        )
+    return nodes, source